Rework subscribing to deleted channels
This commit is contained in:
parent
15e3f06ad6
commit
109dcd22de
|
@ -13,6 +13,16 @@ module.exports = [
|
||||||
const data = await fetchChannel(id, settings.instance)
|
const data = await fetchChannel(id, settings.instance)
|
||||||
const subscribed = user.isSubscribed(id)
|
const subscribed = user.isSubscribed(id)
|
||||||
const instanceOrigin = settings.instance
|
const instanceOrigin = settings.instance
|
||||||
|
|
||||||
|
// problem with the channel? fetchChannel has collected the necessary information for us.
|
||||||
|
// we can render a skeleton page, display the message, and provide the option to unsubscribe.
|
||||||
|
if (data.error) {
|
||||||
|
const statusCode = data.missing ? 410 : 500
|
||||||
|
return render(statusCode, "pug/channel-error.pug", {settings, data, subscribed, instanceOrigin})
|
||||||
|
}
|
||||||
|
|
||||||
|
// everything is fine
|
||||||
|
|
||||||
// normalise info, apply watched status
|
// normalise info, apply watched status
|
||||||
if (!data.second__subCountText && data.subCount) {
|
if (!data.second__subCountText && data.subCount) {
|
||||||
data.second__subCountText = converters.subscriberCountToText(data.subCount)
|
data.second__subCountText = converters.subscriberCountToText(data.subCount)
|
||||||
|
@ -24,7 +34,7 @@ module.exports = [
|
||||||
video.watched = watchedVideos.includes(video.videoId)
|
video.watched = watchedVideos.includes(video.videoId)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return render(200, "pug/channel.pug", {settings, url, data, subscribed, instanceOrigin})
|
return render(200, "pug/channel.pug", {settings, data, subscribed, instanceOrigin})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
@ -26,7 +26,6 @@ module.exports = [
|
||||||
await fetchChannel(ucid, settings.instance)
|
await fetchChannel(ucid, settings.instance)
|
||||||
db.prepare(
|
db.prepare(
|
||||||
"INSERT INTO Subscriptions (token, ucid) VALUES (?, ?)"
|
"INSERT INTO Subscriptions (token, ucid) VALUES (?, ?)"
|
||||||
+ " ON CONFLICT (token, ucid) DO UPDATE SET channel_missing = 0"
|
|
||||||
).run(token, ucid)
|
).run(token, ucid)
|
||||||
} else {
|
} else {
|
||||||
db.prepare("DELETE FROM Subscriptions WHERE token = ? AND ucid = ?").run(token, ucid)
|
db.prepare("DELETE FROM Subscriptions WHERE token = ? AND ucid = ?").run(token, ucid)
|
||||||
|
@ -41,7 +40,6 @@ module.exports = [
|
||||||
}),
|
}),
|
||||||
content: "Success, redirecting..."
|
content: "Success, redirecting..."
|
||||||
}
|
}
|
||||||
return redirect(params.get("referrer"), 303)
|
|
||||||
} else {
|
} else {
|
||||||
return {
|
return {
|
||||||
statusCode: 200,
|
statusCode: 200,
|
||||||
|
|
|
@ -11,12 +11,14 @@ module.exports = [
|
||||||
let hasSubscriptions = false
|
let hasSubscriptions = false
|
||||||
let videos = []
|
let videos = []
|
||||||
let channels = []
|
let channels = []
|
||||||
|
let missingChannelCount = 0
|
||||||
let refreshed = null
|
let refreshed = null
|
||||||
if (user.token) {
|
if (user.token) {
|
||||||
// trigger a background refresh, needed if they came back from being inactive
|
// trigger a background refresh, needed if they came back from being inactive
|
||||||
refresher.skipWaiting()
|
refresher.skipWaiting()
|
||||||
// get channels
|
// get channels
|
||||||
channels = db.prepare(`SELECT Channels.* FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ? ORDER BY name`).all(user.token)
|
channels = db.prepare(`SELECT Channels.* FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ? ORDER BY name`).all(user.token)
|
||||||
|
missingChannelCount = channels.reduce((a, c) => a + c.missing, 0)
|
||||||
// get refreshed status
|
// get refreshed status
|
||||||
refreshed = db.prepare(`SELECT min(refreshed) as min, max(refreshed) as max, count(refreshed) as count FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ?`).get(user.token)
|
refreshed = db.prepare(`SELECT min(refreshed) as min, max(refreshed) as max, count(refreshed) as count FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ?`).get(user.token)
|
||||||
// get watched videos
|
// get watched videos
|
||||||
|
@ -37,7 +39,7 @@ module.exports = [
|
||||||
}
|
}
|
||||||
const settings = user.getSettingsOrDefaults()
|
const settings = user.getSettingsOrDefaults()
|
||||||
const instanceOrigin = settings.instance
|
const instanceOrigin = settings.instance
|
||||||
return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin})
|
return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, missingChannelCount, refreshed, timeToPastText, instanceOrigin})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
|
@ -15,8 +15,8 @@ const prepared = {
|
||||||
channel_refreshed_update: db.prepare(
|
channel_refreshed_update: db.prepare(
|
||||||
"UPDATE Channels SET refreshed = ? WHERE ucid = ?"
|
"UPDATE Channels SET refreshed = ? WHERE ucid = ?"
|
||||||
),
|
),
|
||||||
unsubscribe_all_from_channel: db.prepare(
|
channel_mark_as_missing: db.prepare(
|
||||||
"UPDATE Subscriptions SET channel_missing = 1 WHERE ucid = ?"
|
"UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?"
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,7 +35,7 @@ class RefreshQueue {
|
||||||
// get the next set of scheduled channels to refresh
|
// get the next set of scheduled channels to refresh
|
||||||
const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
|
const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
|
||||||
const channels = db.prepare(
|
const channels = db.prepare(
|
||||||
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? WHERE Subscriptions.channel_missing = 0 ORDER BY SeenTokens.seen DESC"
|
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token INNER JOIN Channels ON Channels.ucid = Subscriptions.ucid WHERE Channels.missing = 0 AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
|
||||||
).pluck().all(afterTime)
|
).pluck().all(afterTime)
|
||||||
this.addLast(channels)
|
this.addLast(channels)
|
||||||
this.lastLoadTime = Date.now()
|
this.lastLoadTime = Date.now()
|
||||||
|
@ -72,11 +72,12 @@ class Refresher {
|
||||||
this.refreshQueue = new RefreshQueue()
|
this.refreshQueue = new RefreshQueue()
|
||||||
this.state = this.sym.ACTIVE
|
this.state = this.sym.ACTIVE
|
||||||
this.waitingTimeout = null
|
this.waitingTimeout = null
|
||||||
|
this.lastFakeNotFoundTime = 0
|
||||||
this.next()
|
this.next()
|
||||||
}
|
}
|
||||||
|
|
||||||
refreshChannel(ucid) {
|
refreshChannel(ucid) {
|
||||||
return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
|
return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(/** @param {any} root */ root => {
|
||||||
if (Array.isArray(root)) {
|
if (Array.isArray(root)) {
|
||||||
root.forEach(video => {
|
root.forEach(video => {
|
||||||
// organise
|
// organise
|
||||||
|
@ -89,11 +90,24 @@ class Refresher {
|
||||||
prepared.channel_refreshed_update.run(Date.now(), ucid)
|
prepared.channel_refreshed_update.run(Date.now(), ucid)
|
||||||
// console.log(`updated ${root.length} videos for channel ${ucid}`)
|
// console.log(`updated ${root.length} videos for channel ${ucid}`)
|
||||||
} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
|
} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
|
||||||
return [] // nothing we can do. skip this iteration.
|
// nothing we can do. skip this iteration.
|
||||||
} else if (root.identifier === "NOT_FOUND") {
|
} else if (root.identifier === "NOT_FOUND") {
|
||||||
// the channel does not exist. we should unsubscribe all users so we don't try again.
|
// YouTube sometimes returns not found for absolutely no reason.
|
||||||
// console.log(`channel ${ucid} does not exist, unsubscribing all users`)
|
// There is no way to distinguish between a fake missing channel and a real missing channel without requesting the real endpoint.
|
||||||
prepared.unsubscribe_all_from_channel.run(ucid)
|
// These fake missing channels often happen in bursts, which is why there is a cooldown.
|
||||||
|
const timeSinceLastFakeNotFound = Date.now() - this.lastFakeNotFoundTime
|
||||||
|
if (timeSinceLastFakeNotFound >= constants.caching.subscriptions_refesh_fake_not_found_cooldown) {
|
||||||
|
// We'll request the real endpoint to verify.
|
||||||
|
fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}`).then(res => res.json()).then(/** @param {any} root */ root => {
|
||||||
|
if (root.error && (root.identifier === "NOT_FOUND" || root.identifier === "ACCOUNT_TERMINATED")) {
|
||||||
|
// The channel is really gone, and we should mark it as missing for everyone.
|
||||||
|
prepared.channel_mark_as_missing.run(root.error, ucid)
|
||||||
|
} else {
|
||||||
|
// The channel is not actually gone and YouTube is trolling us.
|
||||||
|
this.lastFakeNotFoundTime = Date.now()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} // else youtube is currently trolling us, skip this until later.
|
||||||
} else {
|
} else {
|
||||||
throw new Error(root.error)
|
throw new Error(root.error)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
extends includes/layout
|
||||||
|
|
||||||
|
include includes/video-list-item
|
||||||
|
include includes/subscribe-button
|
||||||
|
|
||||||
|
block head
|
||||||
|
title= `${data.row ? data.row.name : "Deleted channel"} - CloudTube`
|
||||||
|
script(type="module" src=getStaticURL("html", "/static/js/channel.js"))
|
||||||
|
|
||||||
|
block content
|
||||||
|
main.channel-page
|
||||||
|
if data.row
|
||||||
|
.channel-data
|
||||||
|
.info
|
||||||
|
- const iconURL = data.row.icon_url
|
||||||
|
if iconURL
|
||||||
|
.logo
|
||||||
|
img(src=iconURL alt="").thumbnail-image
|
||||||
|
.about
|
||||||
|
h1.name= data.row.name
|
||||||
|
+subscribe_button(data.ucid, subscribed, `/channel/${data.ucid}`).subscribe-button.base-border-look
|
||||||
|
|
||||||
|
.channel-error
|
||||||
|
div= data.message
|
||||||
|
|
||||||
|
if data.missing && subscribed
|
||||||
|
.you-should-unsubscribe To remove this channel from your subscriptions list, click Unsubscribe.
|
|
@ -11,12 +11,24 @@ block content
|
||||||
if hasSubscriptions
|
if hasSubscriptions
|
||||||
section
|
section
|
||||||
details.channels-details
|
details.channels-details
|
||||||
summary #{channels.length} subscriptions
|
summary
|
||||||
|
| #{channels.length} subscriptions
|
||||||
|
if missingChannelCount === 1
|
||||||
|
= ` - ${missingChannelCount} channel is gone`
|
||||||
|
else if missingChannelCount > 1
|
||||||
|
= ` - ${missingChannelCount} channels are gone`
|
||||||
.channels-list
|
.channels-list
|
||||||
for channel in channels
|
for channel in channels
|
||||||
a(href=`/channel/${channel.ucid}`).channel-item
|
a(href=`/channel/${channel.ucid}`).channel-item
|
||||||
img(src=channel.icon_url width=512 height=512 alt="").thumbnail
|
img(src=channel.icon_url width=512 height=512 alt="").thumbnail
|
||||||
span.name= channel.name
|
div
|
||||||
|
div.name= channel.name
|
||||||
|
if channel.missing
|
||||||
|
div.missing-reason
|
||||||
|
if channel.missing_reason
|
||||||
|
= channel.missing_reason
|
||||||
|
else
|
||||||
|
| This channel appears to be deleted or terminated. Click to check it.
|
||||||
|
|
||||||
if refreshed
|
if refreshed
|
||||||
section
|
section
|
||||||
|
|
|
@ -74,6 +74,19 @@ $_theme: () !default
|
||||||
.channel-video
|
.channel-video
|
||||||
@include channel-video
|
@include channel-video
|
||||||
|
|
||||||
|
.channel-error
|
||||||
|
background-color: map.get($_theme, "bg-1")
|
||||||
|
padding: 24px
|
||||||
|
margin: 12px 0px 24px
|
||||||
|
border-radius: 8px
|
||||||
|
border: 1px solid map.get($_theme, "edge-grey")
|
||||||
|
font-size: 20px
|
||||||
|
color: map.get($_theme, "fg-warning")
|
||||||
|
|
||||||
|
.you-should-unsubscribe
|
||||||
|
margin-top: 20px
|
||||||
|
color: map.get($_theme, "fg-main")
|
||||||
|
|
||||||
.about-description // class provided by youtube
|
.about-description // class provided by youtube
|
||||||
pre
|
pre
|
||||||
font-size: inherit
|
font-size: inherit
|
||||||
|
|
|
@ -37,6 +37,10 @@ $_theme: () !default
|
||||||
font-size: 22px
|
font-size: 22px
|
||||||
color: map.get($_theme, "fg-main")
|
color: map.get($_theme, "fg-main")
|
||||||
|
|
||||||
|
.missing-reason
|
||||||
|
font-size: 16px
|
||||||
|
color: map.get($_theme, "fg-warning")
|
||||||
|
|
||||||
@include forms.checkbox-hider("watched-videos-display")
|
@include forms.checkbox-hider("watched-videos-display")
|
||||||
|
|
||||||
#watched-videos-display:checked ~ .video-list-item--watched
|
#watched-videos-display:checked ~ .video-list-item--watched
|
||||||
|
|
|
@ -50,6 +50,7 @@ let constants = {
|
||||||
csrf_time: 4*60*60*1000,
|
csrf_time: 4*60*60*1000,
|
||||||
seen_token_subscriptions_eligible: 40*60*60*1000,
|
seen_token_subscriptions_eligible: 40*60*60*1000,
|
||||||
subscriptions_refresh_loop_min: 5*60*1000,
|
subscriptions_refresh_loop_min: 5*60*1000,
|
||||||
|
subscriptions_refesh_fake_not_found_cooldown: 10*60*1000,
|
||||||
},
|
},
|
||||||
|
|
||||||
// Pattern matching.
|
// Pattern matching.
|
||||||
|
|
|
@ -54,7 +54,7 @@ class User {
|
||||||
|
|
||||||
getSubscriptions() {
|
getSubscriptions() {
|
||||||
if (this.token) {
|
if (this.token) {
|
||||||
return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ? AND channel_missing = 0").pluck().all(this.token)
|
return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ?").pluck().all(this.token)
|
||||||
} else {
|
} else {
|
||||||
return []
|
return []
|
||||||
}
|
}
|
||||||
|
@ -62,7 +62,7 @@ class User {
|
||||||
|
|
||||||
isSubscribed(ucid) {
|
isSubscribed(ucid) {
|
||||||
if (this.token) {
|
if (this.token) {
|
||||||
return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ? AND channel_missing = 0").get([this.token, ucid])
|
return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ?").get([this.token, ucid])
|
||||||
} else {
|
} else {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
|
@ -75,6 +75,26 @@ const deltas = [
|
||||||
function() {
|
function() {
|
||||||
db.prepare("ALTER TABLE Settings ADD COLUMN theme INTEGER DEFAULT 0")
|
db.prepare("ALTER TABLE Settings ADD COLUMN theme INTEGER DEFAULT 0")
|
||||||
.run()
|
.run()
|
||||||
|
},
|
||||||
|
// 12: Channels +missing +missing_reason, Subscriptions -
|
||||||
|
// Better management for missing channels
|
||||||
|
// We totally discard the existing Subscriptions.channel_missing since it is unreliable.
|
||||||
|
function() {
|
||||||
|
db.prepare("ALTER TABLE Channels ADD COLUMN missing INTEGER NOT NULL DEFAULT 0")
|
||||||
|
.run()
|
||||||
|
db.prepare("ALTER TABLE Channels ADD COLUMN missing_reason TEXT")
|
||||||
|
.run()
|
||||||
|
// https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes
|
||||||
|
db.transaction(() => {
|
||||||
|
db.prepare("CREATE TABLE NEW_Subscriptions (token TEXT NOT NULL, ucid TEXT NOT NULL, PRIMARY KEY (token, ucid))")
|
||||||
|
.run()
|
||||||
|
db.prepare("INSERT INTO NEW_Subscriptions (token, ucid) SELECT token, ucid FROM Subscriptions")
|
||||||
|
.run()
|
||||||
|
db.prepare("DROP TABLE Subscriptions")
|
||||||
|
.run()
|
||||||
|
db.prepare("ALTER TABLE NEW_Subscriptions RENAME TO Subscriptions")
|
||||||
|
.run()
|
||||||
|
})()
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
@ -2,14 +2,58 @@ const {request} = require("./request")
|
||||||
const db = require("./db")
|
const db = require("./db")
|
||||||
|
|
||||||
async function fetchChannel(ucid, instance) {
|
async function fetchChannel(ucid, instance) {
|
||||||
|
function updateGoodData(channel) {
|
||||||
|
const bestIcon = channel.authorThumbnails.slice(-1)[0]
|
||||||
|
const iconURL = bestIcon ? bestIcon.url : null
|
||||||
|
db.prepare("REPLACE INTO Channels (ucid, name, icon_url, missing, missing_reason) VALUES (?, ?, ?, 0, NULL)").run(channel.authorId, channel.author, iconURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateBadData(channel) {
|
||||||
|
if (channel.identifier === "NOT_FOUND" || channel.identifier === "ACCOUNT_TERMINATED") {
|
||||||
|
db.prepare("UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?").run(channel.error, channel.authorId)
|
||||||
|
return {
|
||||||
|
missing: true,
|
||||||
|
message: channel.error
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
missing: false,
|
||||||
|
message: channel.error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!instance) throw new Error("No instance parameter provided")
|
if (!instance) throw new Error("No instance parameter provided")
|
||||||
// fetch
|
|
||||||
|
const row = db.prepare("SELECT * FROM Channels WHERE ucid = ?").get(ucid)
|
||||||
|
|
||||||
|
// handle the case where the channel has a known error
|
||||||
|
if (row && row.missing_reason) {
|
||||||
|
return {
|
||||||
|
error: true,
|
||||||
|
ucid,
|
||||||
|
row,
|
||||||
|
missing: true,
|
||||||
|
message: row.missing_reason
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @type {any} */
|
||||||
const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json())
|
const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json())
|
||||||
// update database
|
|
||||||
const bestIcon = channel.authorThumbnails.slice(-1)[0]
|
// handle the case where the channel has a newly discovered error
|
||||||
const iconURL = bestIcon ? bestIcon.url : null
|
if (channel.error) {
|
||||||
db.prepare("REPLACE INTO Channels (ucid, name, icon_url) VALUES (?, ?, ?)").run([channel.authorId, channel.author, iconURL])
|
const missingData = updateBadData(channel)
|
||||||
// return
|
return {
|
||||||
|
error: true,
|
||||||
|
ucid,
|
||||||
|
row,
|
||||||
|
...missingData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handle the case where the channel returns good data (this is the only remaining scenario)
|
||||||
|
updateGoodData(channel)
|
||||||
return channel
|
return channel
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue