experiments
/
cloudtube_fork
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rework subscribing to deleted channels

This commit is contained in:
Cadence Ember 2022-01-10 14:18:45 +13:00
parent 15e3f06ad6
commit 109dcd22de
No known key found for this signature in database
GPG Key ID: BC1C2C61CF521B17
12 changed files with 167 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/channels.js
View File

@ -13,6 +13,16 @@ module.exports = [
const data = await fetchChannel(id, settings.instance)
const subscribed = user.isSubscribed(id)
const instanceOrigin = settings.instance
// problem with the channel? fetchChannel has collected the necessary information for us.
// we can render a skeleton page, display the message, and provide the option to unsubscribe.
if (data.error) {
const statusCode = data.missing ? 410 : 500
return render(statusCode, "pug/channel-error.pug", {settings, data, subscribed, instanceOrigin})
}
// everything is fine
// normalise info, apply watched status
if (!data.second__subCountText && data.subCount) {
data.second__subCountText = converters.subscriberCountToText(data.subCount)
@ -24,7 +34,7 @@ module.exports = [
video.watched = watchedVideos.includes(video.videoId)
})
}
return render(200, "pug/channel.pug", {settings, url, data, subscribed, instanceOrigin})
return render(200, "pug/channel.pug", {settings, data, subscribed, instanceOrigin})
}
}
]

2
api/formapi.js
View File

@ -26,7 +26,6 @@ module.exports = [
await fetchChannel(ucid, settings.instance)
db.prepare(
"INSERT INTO Subscriptions (token, ucid) VALUES (?, ?)"
+ " ON CONFLICT (token, ucid) DO UPDATE SET channel_missing = 0"
).run(token, ucid)
} else {
db.prepare("DELETE FROM Subscriptions WHERE token = ? AND ucid = ?").run(token, ucid)
@ -41,7 +40,6 @@ module.exports = [
}),
content: "Success, redirecting..."
}
return redirect(params.get("referrer"), 303)
} else {
return {
statusCode: 200,

4
api/subscriptions.js
View File

@ -11,12 +11,14 @@ module.exports = [
let hasSubscriptions = false
let videos = []
let channels = []
let missingChannelCount = 0
let refreshed = null
if (user.token) {
// trigger a background refresh, needed if they came back from being inactive
refresher.skipWaiting()
// get channels
channels = db.prepare(`SELECT Channels.* FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ? ORDER BY name`).all(user.token)
missingChannelCount = channels.reduce((a, c) => a + c.missing, 0)
// get refreshed status
refreshed = db.prepare(`SELECT min(refreshed) as min, max(refreshed) as max, count(refreshed) as count FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ?`).get(user.token)
// get watched videos
@ -37,7 +39,7 @@ module.exports = [
}
const settings = user.getSettingsOrDefaults()
const instanceOrigin = settings.instance
return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin})
return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, missingChannelCount, refreshed, timeToPastText, instanceOrigin})
}
}
]

30
background/feed-update.js
View File

@ -15,8 +15,8 @@ const prepared = {
channel_refreshed_update: db.prepare(
"UPDATE Channels SET refreshed = ? WHERE ucid = ?"
),
unsubscribe_all_from_channel: db.prepare(
"UPDATE Subscriptions SET channel_missing = 1 WHERE ucid = ?"
channel_mark_as_missing: db.prepare(
"UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?"
)
}
@ -35,7 +35,7 @@ class RefreshQueue {
// get the next set of scheduled channels to refresh
const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
const channels = db.prepare(
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? WHERE Subscriptions.channel_missing = 0 ORDER BY SeenTokens.seen DESC"
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token INNER JOIN Channels ON Channels.ucid = Subscriptions.ucid WHERE Channels.missing = 0 AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
).pluck().all(afterTime)
this.addLast(channels)
this.lastLoadTime = Date.now()
@ -72,11 +72,12 @@ class Refresher {
this.refreshQueue = new RefreshQueue()
this.state = this.sym.ACTIVE
this.waitingTimeout = null
this.lastFakeNotFoundTime = 0
this.next()
}
refreshChannel(ucid) {
return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(/** @param {any} root */ root => {
if (Array.isArray(root)) {
root.forEach(video => {
// organise
@ -89,11 +90,24 @@ class Refresher {
prepared.channel_refreshed_update.run(Date.now(), ucid)
// console.log(`updated ${root.length} videos for channel ${ucid}`)
} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
return [] // nothing we can do. skip this iteration.
// nothing we can do. skip this iteration.
} else if (root.identifier === "NOT_FOUND") {
// the channel does not exist. we should unsubscribe all users so we don't try again.
// console.log(`channel ${ucid} does not exist, unsubscribing all users`)
prepared.unsubscribe_all_from_channel.run(ucid)
// YouTube sometimes returns not found for absolutely no reason.
// There is no way to distinguish between a fake missing channel and a real missing channel without requesting the real endpoint.
// These fake missing channels often happen in bursts, which is why there is a cooldown.
const timeSinceLastFakeNotFound = Date.now() - this.lastFakeNotFoundTime
if (timeSinceLastFakeNotFound >= constants.caching.subscriptions_refesh_fake_not_found_cooldown) {
// We'll request the real endpoint to verify.
fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}`).then(res => res.json()).then(/** @param {any} root */ root => {
if (root.error && (root.identifier === "NOT_FOUND" || root.identifier === "ACCOUNT_TERMINATED")) {
// The channel is really gone, and we should mark it as missing for everyone.
prepared.channel_mark_as_missing.run(root.error, ucid)
} else {
// The channel is not actually gone and YouTube is trolling us.
this.lastFakeNotFoundTime = Date.now()
}
})
} // else youtube is currently trolling us, skip this until later.
} else {
throw new Error(root.error)
}

27
pug/channel-error.pug Normal file
View File

@ -0,0 +1,27 @@
extends includes/layout
include includes/video-list-item
include includes/subscribe-button
block head
title= `${data.row ? data.row.name : "Deleted channel"} - CloudTube`
script(type="module" src=getStaticURL("html", "/static/js/channel.js"))
block content
main.channel-page
if data.row
.channel-data
.info
- const iconURL = data.row.icon_url
if iconURL
.logo
img(src=iconURL alt="").thumbnail-image
.about
h1.name= data.row.name
+subscribe_button(data.ucid, subscribed, `/channel/${data.ucid}`).subscribe-button.base-border-look
.channel-error
div= data.message
if data.missing && subscribed
.you-should-unsubscribe To remove this channel from your subscriptions list, click Unsubscribe.

16
pug/subscriptions.pug
View File

@ -11,12 +11,24 @@ block content
if hasSubscriptions
section
details.channels-details
summary #{channels.length} subscriptions
summary
| #{channels.length} subscriptions
if missingChannelCount === 1
= ` - ${missingChannelCount} channel is gone`
else if missingChannelCount > 1
= ` - ${missingChannelCount} channels are gone`
.channels-list
for channel in channels
a(href=`/channel/${channel.ucid}`).channel-item
img(src=channel.icon_url width=512 height=512 alt="").thumbnail
span.name= channel.name
div
div.name= channel.name
if channel.missing
div.missing-reason
if channel.missing_reason
= channel.missing_reason
else
| This channel appears to be deleted or terminated. Click to check it.
if refreshed
section

13
sass/includes/_channel-page.sass
View File

@ -74,6 +74,19 @@ $_theme: () !default
.channel-video
@include channel-video
.channel-error
background-color: map.get($_theme, "bg-1")
padding: 24px
margin: 12px 0px 24px
border-radius: 8px
border: 1px solid map.get($_theme, "edge-grey")
font-size: 20px
color: map.get($_theme, "fg-warning")
.you-should-unsubscribe
margin-top: 20px
color: map.get($_theme, "fg-main")
.about-description // class provided by youtube
pre
font-size: inherit

4
sass/includes/_subscriptions-page.sass
View File

@ -37,6 +37,10 @@ $_theme: () !default
font-size: 22px
color: map.get($_theme, "fg-main")
.missing-reason
font-size: 16px
color: map.get($_theme, "fg-warning")
@include forms.checkbox-hider("watched-videos-display")
#watched-videos-display:checked ~ .video-list-item--watched

1
utils/constants.js
View File

@ -50,6 +50,7 @@ let constants = {
csrf_time: 4*60*60*1000,
seen_token_subscriptions_eligible: 40*60*60*1000,
subscriptions_refresh_loop_min: 5*60*1000,
subscriptions_refesh_fake_not_found_cooldown: 10*60*1000,
},
// Pattern matching.

4
utils/getuser.js
View File

@ -54,7 +54,7 @@ class User {
getSubscriptions() {
if (this.token) {
return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ? AND channel_missing = 0").pluck().all(this.token)
return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ?").pluck().all(this.token)
} else {
return []
}
@ -62,7 +62,7 @@ class User {
isSubscribed(ucid) {
if (this.token) {
return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ? AND channel_missing = 0").get([this.token, ucid])
return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ?").get([this.token, ucid])
} else {
return false
}

20
utils/upgradedb.js
View File

@ -75,6 +75,26 @@ const deltas = [
function() {
db.prepare("ALTER TABLE Settings ADD COLUMN theme INTEGER DEFAULT 0")
.run()
},
// 12: Channels +missing +missing_reason, Subscriptions -
// Better management for missing channels
// We totally discard the existing Subscriptions.channel_missing since it is unreliable.
function() {
db.prepare("ALTER TABLE Channels ADD COLUMN missing INTEGER NOT NULL DEFAULT 0")
.run()
db.prepare("ALTER TABLE Channels ADD COLUMN missing_reason TEXT")
.run()
// https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes
db.transaction(() => {
db.prepare("CREATE TABLE NEW_Subscriptions (token TEXT NOT NULL, ucid TEXT NOT NULL, PRIMARY KEY (token, ucid))")
.run()
db.prepare("INSERT INTO NEW_Subscriptions (token, ucid) SELECT token, ucid FROM Subscriptions")
.run()
db.prepare("DROP TABLE Subscriptions")
.run()
db.prepare("ALTER TABLE NEW_Subscriptions RENAME TO Subscriptions")
.run()
})()
}
]

56
utils/youtube.js
View File

@ -2,14 +2,58 @@ const {request} = require("./request")
const db = require("./db")
async function fetchChannel(ucid, instance) {
if (!instance) throw new Error("No instance parameter provided")
// fetch
const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json())
// update database
function updateGoodData(channel) {
const bestIcon = channel.authorThumbnails.slice(-1)[0]
const iconURL = bestIcon ? bestIcon.url : null
db.prepare("REPLACE INTO Channels (ucid, name, icon_url) VALUES (?, ?, ?)").run([channel.authorId, channel.author, iconURL])
// return
db.prepare("REPLACE INTO Channels (ucid, name, icon_url, missing, missing_reason) VALUES (?, ?, ?, 0, NULL)").run(channel.authorId, channel.author, iconURL)
}
function updateBadData(channel) {
if (channel.identifier === "NOT_FOUND" || channel.identifier === "ACCOUNT_TERMINATED") {
db.prepare("UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?").run(channel.error, channel.authorId)
return {
missing: true,
message: channel.error
}
} else {
return {
missing: false,
message: channel.error
}
}
}
if (!instance) throw new Error("No instance parameter provided")
const row = db.prepare("SELECT * FROM Channels WHERE ucid = ?").get(ucid)
// handle the case where the channel has a known error
if (row && row.missing_reason) {
return {
error: true,
ucid,
row,
missing: true,
message: row.missing_reason
}
}
/** @type {any} */
const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json())
// handle the case where the channel has a newly discovered error
if (channel.error) {
const missingData = updateBadData(channel)
return {
error: true,
ucid,
row,
...missingData
}
}
// handle the case where the channel returns good data (this is the only remaining scenario)
updateGoodData(channel)
return channel
}