@@ -95,7 +95,7 @@ def template_mix(mix, listen)
mix["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr
index 7c584d15..955e0855 100644
--- a/src/invidious/playlists.cr
+++ b/src/invidious/playlists.cr
@@ -46,14 +46,8 @@ struct PlaylistVideo
XML.build { |xml| to_xml(xml) }
end
- def to_json(locale : String?, json : JSON::Builder)
- to_json(json)
- end
-
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
- json.field "type", "video"
-
json.field "title", self.title
json.field "videoId", self.id
@@ -73,7 +67,6 @@ struct PlaylistVideo
end
json.field "lengthSeconds", self.length_seconds
- json.field "liveNow", self.live_now
end
end
@@ -270,7 +263,7 @@ end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
- title: playlist.title[..150],
+ title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
@@ -373,8 +366,6 @@ def fetch_playlist(plid : String)
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
- elsif text.includes? "episode"
- video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
@@ -432,7 +423,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
- videos = [] of PlaylistVideo | ProblematicTimelineItem
+ videos = [] of PlaylistVideo
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
@@ -448,7 +439,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32,
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
- videos = [] of PlaylistVideo | ProblematicTimelineItem
+ videos = [] of PlaylistVideo
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@@ -500,14 +491,12 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
index: index,
})
end
- rescue ex
- videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
-def template_playlist(playlist, listen)
+def template_playlist(playlist)
html = <<-END_HTML
@@ -521,7 +510,7 @@ def template_playlist(playlist, listen)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index c8db207c..5b55738e 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -53,7 +53,7 @@ module Invidious::Routes::Account
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
+ new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -123,10 +123,8 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
- PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)
@@ -240,7 +238,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
+ scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -328,9 +326,17 @@ module Invidious::Routes::Account
end
end
- case action = env.params.query["action"]?
- when "revoke_token"
- session = env.params.query["session"]
+ if env.params.query["action_revoke_token"]?
+ action = "action_revoke_token"
+ else
+ return env.redirect referer
+ end
+
+ session = env.params.query["session"]?
+ session ||= ""
+
+ case action
+ when .starts_with? "action_revoke_token"
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index c27caad7..d89e752c 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,11 +8,6 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
- end
-
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -32,21 +27,28 @@ module Invidious::Routes::API::Manifest
haltf env, status_code: response.status_code
end
- # Proxy URLs for video playback on invidious.
- # Other API clients can get the original URLs by omiting `local=true`.
- manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("").rchop(" ")
- url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
+ manifest = response.body
+
+ manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("")
+ url = url.rchop(" ")
+
+ if local
+ uri = URI.parse(url)
+ url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
+ end
+
"#{url} "
end
return manifest
end
- # Ditto, only proxify URLs if `local=true` is used
+ adaptive_fmts = video.adaptive_fmts
+
if local
- video.adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
+ adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
end
@@ -68,23 +70,17 @@ module Invidious::Routes::API::Manifest
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
- audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
- lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
- is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
- displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
- bitrate = fmt["bitrate"]
-
# Different representations of the same audio should be groupped into one AdaptationSet.
# However, most players don't support auto quality switching, so we have to trick them
# into providing a quality selector.
# See https://github.com/iv-org/invidious/issues/3074 for more details.
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
- xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
@@ -181,9 +177,8 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
- uri = URI.parse(match)
- path = uri.path
+ manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
+ path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -212,7 +207,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["host"] = uri.host.not_nil!
+ raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index a940ee68..7faf200a 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -27,21 +27,10 @@ module Invidious::Routes::API::V1::Channels
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ rescue ex
+ return error_json(500, ex)
end
JSON.build do |json|
@@ -95,7 +84,6 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
- json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
@@ -154,23 +142,12 @@ module Invidious::Routes::API::V1::Channels
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -197,26 +174,14 @@ module Invidious::Routes::API::V1::Channels
get_channel()
# Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -243,26 +208,14 @@ module Invidious::Routes::API::V1::Channels
get_channel()
# Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -368,35 +321,6 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.courses(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
def self.community(env)
locale = env.get("preferences").as(Preferences).locale
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index fea2993c..41865f34 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 403, error_message
+ haltf env, 400, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 4f5b58da..12942906 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -42,9 +42,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -77,9 +74,7 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a.empty?
- json_response = JSON.parse(response)
- elsif json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -88,7 +83,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response, listen)
+ playlist_html = template_playlist(json_response)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -114,9 +109,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -147,7 +139,9 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ json.array do
+ Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ end
end
json.field "index", video.index
@@ -161,7 +155,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response, listen)
+ playlist_html = template_mix(response)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -183,8 +177,8 @@ module Invidious::Routes::API::V1::Misc
begin
resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"]
- page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
- if page_type == "WEB_PAGE_TYPE_UNKNOWN"
+ pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
+ if pageType == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url")
end
@@ -200,7 +194,7 @@ module Invidious::Routes::API::V1::Misc
json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
json.field "params", params.try &.as_s
- json.field "pageType", page_type
+ json.field "pageType", pageType
end
end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 59a30745..2922b060 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,7 +31,9 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
+ client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
+ client.before_request { |r| add_yt_headers(r) }
+
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 6a3eb8ae..9281f4dd 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,5 +1,3 @@
-require "html"
-
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -91,14 +89,9 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+ initial_data = YoutubeAPI.get_transcript(params)
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params),
- caption.language_code,
- caption.auto_generated
- )
-
- webvtt = transcript.to_vtt
+ webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
else
# Timedtext API handling
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
@@ -118,7 +111,7 @@ module Invidious::Routes::API::V1::Videos
else
caption_xml = XML.parse(caption_xml)
- webvtt = WebVTT.build(settings_field) do |builder|
+ webvtt = WebVTT.build(settings_field) do |webvtt|
caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds
@@ -138,16 +131,12 @@ module Invidious::Routes::API::V1::Videos
text = "#{md["text"]} "
end
- builder.cue(start_time, end_time, text)
+ webvtt.cue(start_time, end_time, text)
end
end
end
else
- uri = URI.parse(url)
- query_params = uri.query_params
- query_params["fmt"] = "vtt"
- uri.query_params = query_params
- webvtt = YT_POOL.client &.get(uri.request_target).body
+ webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
if webvtt.starts_with?(" 0
url += "?#{env.params.query}"
@@ -74,15 +72,13 @@ module Invidious::Routes::Embed
url = "/playlist?list=#{plid}"
raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video.id}"
+ url = "/embed/#{videos[0].id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -161,12 +157,10 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -207,14 +201,6 @@ module Invidious::Routes::Embed
return env.redirect url
end
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
rendered "embed"
end
end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..e20a7139 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -143,25 +143,32 @@ module Invidious::Routes::Feeds
# RSS feeds
def self.rss_channel(env)
+ locale = env.get("preferences").as(Preferences).locale
+
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- if env.params.url["ucid"].matches?(/^[\w-]+$/)
- ucid = env.params.url["ucid"]
- else
- return error_atom(400, InfoException.new("Invalid channel ucid provided."))
- end
+ ucid = env.params.url["ucid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_atom(404, ex)
+ rescue ex
+ return error_atom(500, ex)
+ end
+
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
- return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@@ -172,7 +179,7 @@ module Invidious::Routes::Feeds
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
@@ -180,44 +187,43 @@ module Invidious::Routes::Feeds
title: title,
id: video_id,
author: author,
- ucid: video_ucid,
+ ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
+ live_now: false,
+ paid: false,
+ premium: false,
premiere_timestamp: nil,
author_verified: false,
- author_thumbnail: nil,
- badges: VideoBadges::None,
})
end
- author = ""
- author = videos[0].author if videos.size > 0
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{ucid}" }
- xml.element("yt:channelId") { xml.text ucid }
- xml.element("title") { xml.text author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
+ xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
+ xml.element("yt:channelId") { xml.text channel.ucid }
+ xml.element("icon") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
- xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ xml.element("name") { xml.text channel.author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
xml.element("image") do
- xml.element("url") { xml.text "" }
- xml.element("title") { xml.text author }
+ xml.element("url") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(false, params, xml)
+ video.to_xml(channel.auto_generated, params, xml)
end
end
end
@@ -296,13 +302,7 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each do |video|
- if video.is_a? PlaylistVideo
- video.to_xml(xml)
- else
- video.to_xml(env, locale, xml)
- end
- end
+ videos.each &.to_xml(xml)
end
end
else
@@ -311,9 +311,8 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
-
document = XML.parse(response.body)
+
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -426,6 +425,16 @@ module Invidious::Routes::Feeds
next # skip this video since it raised an exception (e.g. it is a scheduled live event)
end
+ if CONFIG.enable_user_notifications
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => video.ucid,
+ "videoId" => video.id,
+ "published" => published.to_unix,
+ }.to_json
+ PG_DB.exec("NOTIFY notifications, E'#{payload}'")
+ end
+
video = ChannelVideo.new({
id: id,
title: video.title,
@@ -441,7 +450,11 @@ module Invidious::Routes::Feeds
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
if was_insert
- NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
+ if CONFIG.enable_user_notifications
+ Invidious::Database::Users.add_notification(video)
+ else
+ Invidious::Database::Users.feed_needs_update(video)
+ end
end
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 51d85dfe..b6a2e110 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -11,9 +11,29 @@ module Invidious::Routes::Images
end
end
+ # We're encapsulating this into a proc in order to easily reuse this
+ # portion of the code for each request block below.
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ env.response.headers.delete("Transfer-Encoding")
+ return
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- GGPHT_POOL.client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -41,10 +61,27 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Connection"] = "close"
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool(authority).client &.get(url, headers) do |resp|
- env.response.headers["Connection"] = "close"
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -64,9 +101,26 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool("i9").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -111,7 +165,8 @@ module Invidious::Routes::Images
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
+ # This can likely be optimized into a (small) pool sometime in the future.
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
@@ -126,28 +181,29 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool("i").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
end
-
- private def self.proxy_image(env, response)
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- return proxy_file(response, env)
- end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index e7de5018..add9f75d 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -21,6 +21,9 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
+ captcha_type = env.params.query["captcha"]?
+ captcha_type ||= "image"
+
templated "user/login"
end
@@ -85,14 +88,34 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
+ captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
+ change_type = env.params.body["change_type"]?
- account_type = "invidious"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ if !captcha_type || change_type
+ if change_type
+ captcha_type = change_type
+ end
+ captcha_type ||= "image"
+
+ account_type = "invidious"
+
+ if captcha_type == "image"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ else
+ captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
+ end
+
+ return templated "user/login"
+ end
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- if answer
+ answer ||= ""
+ captcha_type ||= "image"
+
+ case captcha_type
+ when "image"
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -101,8 +124,27 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else
- return templated "user/login"
+ else # "text"
+ answer = Digest::MD5.hexdigest(answer.downcase.strip)
+
+ if tokens.empty?
+ return error_template(500, "Erroneous CAPTCHA")
+ end
+
+ found_valid_captcha = false
+ error_exception = Exception.new
+ tokens.each do |tok|
+ begin
+ validate_request(tok, answer, env.request, HMAC_KEY, locale)
+ found_valid_captcha = true
+ rescue ex
+ error_exception = ex
+ end
+ end
+
+ if !found_valid_captcha
+ return error_template(500, error_exception)
+ end
end
end
@@ -118,9 +160,6 @@ module Invidious::Routes::Login
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
-
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
if env.request.cookies["PREFS"]?
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 0b868755..d6bd9571 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,21 +40,7 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
-
- instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- # Filter out the current instance
- other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
-
- if other_available_instances.empty?
- # If the current instance is the only one, use the redirect URL as fallback
- instance_url = "redirect.invidious.io"
- else
- # Select other random instance
- # Sample returns an array
- # Instances are packaged as {region, domain} in the instance list
- instance_url = other_available_instances.sample(1)[0][1]
- end
-
+ instance_url = fetch_random_instance
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index f2213da4..9c6843e9 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -304,6 +304,23 @@ module Invidious::Routes::Playlists
end
end
+ if env.params.query["action_create_playlist"]?
+ action = "action_create_playlist"
+ elsif env.params.query["action_delete_playlist"]?
+ action = "action_delete_playlist"
+ elsif env.params.query["action_edit_playlist"]?
+ action = "action_edit_playlist"
+ elsif env.params.query["action_add_video"]?
+ action = "action_add_video"
+ video_id = env.params.query["video_id"]
+ elsif env.params.query["action_remove_video"]?
+ action = "action_remove_video"
+ elsif env.params.query["action_move_video_before"]?
+ action = "action_move_video_before"
+ else
+ return env.redirect referer
+ end
+
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
@@ -318,8 +335,12 @@ module Invidious::Routes::Playlists
end
end
- case action = env.params.query["action"]?
- when "add_video"
+ email = user.email
+
+ case action
+ when "action_edit_playlist"
+ # TODO: Playlist stub
+ when "action_add_video"
if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
@@ -356,14 +377,12 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "remove_video"
+ when "action_remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "move_video_before"
+ when "action_move_video_before"
# TODO: Playlist stub
- when nil
- return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 39ca77c0..112535bd 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,10 +27,6 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
- preload = env.params.body["preload"]?.try &.as(String)
- preload ||= "off"
- preload = preload == "on"
-
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -148,7 +144,6 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
- preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -219,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
File.write("config/config.yml", CONFIG.to_yaml)
end
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index b195c7b3..5be33533 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -51,18 +51,8 @@ module Invidious::Routes::Search
else
user = env.get? "user"
- # An URL was copy/pasted in the search box.
- # Redirect the user to the appropriate page.
- if query.url?
- return env.redirect UrlSanitizer.process(query.text).to_s
- end
-
begin
- if user
- items = query.process(user.as(User))
- else
- items = query.process
- end
+ items = query.process
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 1de655d2..7f9ec592 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,16 +32,24 @@ module Invidious::Routes::Subscriptions
end
end
+ if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
+ action = "action_create_subscription_to_channel"
+ elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
+ action = "action_remove_subscriptions"
+ else
+ return env.redirect referer
+ end
+
channel_id = env.params.query["c"]?
channel_id ||= ""
- case action = env.params.query["action"]?
- when "create_subscription_to_channel"
+ case action
+ when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "remove_subscriptions"
+ when "action_remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 083087a9..ec18f3b8 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
+ if !host.matches?(/[\w-]+.googlevideo.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -37,13 +37,12 @@ module Invidious::Routes::VideoPlayback
# See: https://github.com/iv-org/invidious/issues/3302
range_header = env.request.headers["Range"]?
- sq = query_params["sq"]?
- if range_header.nil? && sq.nil?
+ if range_header.nil?
range_for_head = query_params["range"]? || "0-640"
headers["Range"] = "bytes=#{range_for_head}"
end
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -58,7 +57,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region, force_resolve: true)
+ client = make_client(URI.parse(new_host), region, force_resolve = true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -72,7 +71,7 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
rescue ex
error = ex.message
end
@@ -132,7 +131,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- loop do
+ while true
if !range_end && content_length
range_end = content_length
end
@@ -165,13 +164,10 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
- if title = query_params["title"]?
- url = "#{url}&title=#{URI.encode_www_form(title)}"
- end
-
- env.redirect url
+ env.redirect location
break
end
@@ -200,7 +196,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
end
end
@@ -257,11 +253,6 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
- end
-
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index e777b3f1..aabe8dfc 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -121,12 +121,10 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -192,14 +190,6 @@ module Invidious::Routes::Watch
captions: video.captions
)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
templated "watch"
end
@@ -251,10 +241,18 @@ module Invidious::Routes::Watch
end
end
- case action = env.params.query["action"]?
- when "mark_watched"
+ if env.params.query["action_mark_watched"]?
+ action = "action_mark_watched"
+ elsif env.params.query["action_mark_unwatched"]?
+ action = "action_mark_unwatched"
+ else
+ return env.redirect referer
+ end
+
+ case action
+ when "action_mark_watched"
Invidious::Database::Users.mark_watched(user, id)
- when "mark_unwatched"
+ when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -293,9 +291,6 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
- if CONFIG.invidious_companion.present?
- return error_template(403, "Downloads should be routed through Companion when present")
- end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -325,9 +320,10 @@ module Invidious::Routes::Watch
env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i.to_s
+ elsif itag = download_widget["itag"]?.try &.as_i
# URL params specific to /latest_version
env.params.query["id"] = video_id
+ env.params.query["itag"] = itag.to_s
env.params.query["title"] = filename
env.params.query["local"] = "true"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 46b71f1f..ba05da19 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -120,10 +120,8 @@ module Invidious::Routing
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
get "/channel/:ucid/releases", Routes::Channels, :releases
- get "/channel/:ucid/courses", Routes::Channels, :courses
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
- get "/channel/:ucid/posts", Routes::Channels, :community
get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
@@ -238,7 +236,6 @@ module Invidious::Routing
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
- get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending
@@ -246,18 +243,17 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
- get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
- get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
- get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
- get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
+
get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
- get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
+
+ {% for route in {"videos", "latest", "playlists", "community", "search"} %}
+ get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
+ get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
+ {% end %}
# Posts
get "/api/v1/post/:id", {{namespace}}::Channels, :post
@@ -275,6 +271,11 @@ module Invidious::Routing
# Authenticated
+ # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
+ #
+ # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+
get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
index bc2715cf..bf968734 100644
--- a/src/invidious/search/filters.cr
+++ b/src/invidious/search/filters.cr
@@ -75,7 +75,7 @@ module Invidious::Search
@type : Type = Type::All,
@duration : Duration = Duration::None,
@features : Features = Features::None,
- @sort : Sort = Sort::Relevance,
+ @sort : Sort = Sort::Relevance
)
end
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb936..10b81c59 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -37,18 +37,18 @@ module Invidious::Search
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
- view_name = "subscriptions_#{sha256(user.email)}"
-
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
- SELECT *,
- to_tsvector(#{view_name}.title) ||
- to_tsvector(#{view_name}.author)
- as document
- FROM #{view_name}
- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
- query.text, (query.page - 1) * 20,
+ SELECT cv.*,
+ to_tsvector(cv.title) ||
+ to_tsvector(cv.author) AS document
+ FROM channel_videos cv
+ JOIN users ON cv.ucid = any(users.subscriptions)
+ WHERE users.email = $1 AND published > now() - interval '1 month'
+ ORDER BY published
+ ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;",
+ user.email, query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
index 94a92e23..e38845d9 100644
--- a/src/invidious/search/query.cr
+++ b/src/invidious/search/query.cr
@@ -20,9 +20,6 @@ module Invidious::Search
property region : String?
property channel : String = ""
- # Flag that indicates if the smart search features have been disabled.
- @inhibit_ssf : Bool = false
-
# Return true if @raw_query is either `nil` or empty
private def empty_raw_query?
return @raw_query.empty?
@@ -47,22 +44,14 @@ module Invidious::Search
def initialize(
params : HTTP::Params,
@type : Type = Type::Regular,
- @region : String? = nil,
+ @region : String? = nil
)
# Get the raw search query string (common to all search types). In
# Regular search mode, also look for the `search_query` URL parameter
- _raw_query = params["q"]?
- _raw_query ||= params["search_query"]? if @type.regular?
- _raw_query ||= ""
-
- # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
- @raw_query = _raw_query.strip
-
- # Check for smart features (ex: URL search) inhibitor (backslash).
- # If inhibitor is present, remove it.
- if @raw_query.starts_with?('\\')
- @inhibit_ssf = true
- @raw_query = @raw_query[1..]
+ if @type.regular?
+ @raw_query = params["q"]? || params["search_query"]? || ""
+ else
+ @raw_query = params["q"]? || ""
end
# Get the page number (also common to all search types)
@@ -96,7 +85,7 @@ module Invidious::Search
@filters = Filters.from_iv_params(params)
@channel = params["channel"]? || ""
- if @filters.default? && @raw_query.index(/\w:\w/)
+ if @filters.default? && @raw_query.includes?(':')
# Parse legacy filters from query
@filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
else
@@ -147,22 +136,5 @@ module Invidious::Search
return params
end
-
- # Checks if the query is a standalone URL
- def url? : Bool
- # If the smart features have been inhibited, don't go further.
- return false if @inhibit_ssf
-
- # Only supported in regular search mode
- return false if !@type.regular?
-
- # If filters are present, that's a regular search
- return false if !@filters.default?
-
- # Simple heuristics: domain name
- return @raw_query.starts_with?(
- /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
- )
- end
end
end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index d14cde5d..107d148d 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -31,12 +31,12 @@ def fetch_trending(trending_type, region, locale)
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
- extracted.concat itm.contents.select(SearchItem)
+ extracted.concat extract_category(itm)
else
extracted << itm
end
end
# Deduplicate items before returning results
- return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
+ return extracted.select(SearchVideo).uniq!(&.id), plid
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index b175c3b9..8a0f67e5 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,6 +4,8 @@ struct Invidious::User
module Captcha
extend self
+ private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
+
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -58,5 +60,19 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
+
+ def generate_text(key)
+ response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
+ response = JSON.parse(response)
+
+ tokens = response["a"].as_a.map do |answer|
+ generate_response(answer.as_s, {":login"}, key, use_nonce: true)
+ end
+
+ return {
+ question: response["q"].as_s,
+ tokens: tokens,
+ }
+ end
end
end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index 007eb666..108f2ccc 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -115,7 +115,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
+ privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
@@ -124,7 +124,7 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
if idx > CONFIG.playlist_length_limit
raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
@@ -161,7 +161,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def opml?(mimetype : String, extension : String)
+ private def is_opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -179,10 +179,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if opml?(type, extension)
+ if is_opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
+ channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -290,39 +290,42 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
- entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
- return false if entry.nil?
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ io = IO::Memory.new(body)
- begin
- temp = File.tempfile(".db") do |tempfile|
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
- end
+ Compress::Zip::File.open(io) do |file|
+ file.entries.each do |entry|
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- DB.open("sqlite3://" + tempfile.path) do |db|
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
+ next if entry.filename != "newpipe.db"
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
+ tempfile = File.tempfile(".db")
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
- end
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
end
- ensure
- temp.delete if !temp.nil?
+
+ db = DB.open("sqlite3://" + tempfile.path)
+
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
+
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+
+ db.close
+ tempfile.delete
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index 0a8525f3..b3059403 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,7 +4,6 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
- property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 65566d20..0b2d1ef5 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -27,7 +27,6 @@ def get_subscription_feed(user, max_results = 40, page = 1)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
- view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
@@ -53,33 +52,39 @@ def get_subscription_feed(user, max_results = 40, page = 1)
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
+ # "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC"
+ # "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC"
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
else
# Show latest video from each channel
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
-
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
-
- videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
end
end
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 348a0a66..c218b4ef 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -15,7 +15,7 @@ struct Video
# NOTE: don't forget to bump this number if any change is made to
# the `params` structure in videos/parser.cr!!!
#
- SCHEMA_VERSION = 3
+ SCHEMA_VERSION = 2
property id : String
@@ -26,6 +26,12 @@ struct Video
@[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Captions::Metadata
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
+
@[DB::Field(ignore: true)]
property description : String?
@@ -92,24 +98,45 @@ struct Video
# Methods for parsing streaming data
- def fmt_stream : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "formats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def adaptive_fmts : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "adaptiveFormats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -123,8 +150,65 @@ struct Video
# Misc. methods
def storyboards
- container = info.dig?("storyboards") || JSON::Any.new("{}")
- return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
+ storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ if !storyboards
+ if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [{
+ url: storyboard.split("#")[0],
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ storyboard_width: 3,
+ storyboard_height: 3,
+ storyboard_count: -1,
+ }]
+ end
+ end
+
+ items = [] of NamedTuple(
+ url: String,
+ width: Int32,
+ height: Int32,
+ count: Int32,
+ interval: Int32,
+ storyboard_width: Int32,
+ storyboard_height: Int32,
+ storyboard_count: Int32)
+
+ return items if !storyboards
+
+ url = URI.parse(storyboards.shift)
+ params = HTTP::Params.parse(url.query || "")
+
+ storyboards.each_with_index do |sb, i|
+ width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
+ params["sigh"] = sigh
+ url.query = params.to_s
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ storyboard_width = storyboard_width.to_i
+ storyboard_height = storyboard_height.to_i
+ storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
+
+ items << {
+ url: url.to_s.sub("$L", i).sub("$N", "M$M"),
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ storyboard_width: storyboard_width,
+ storyboard_height: storyboard_height,
+ storyboard_count: storyboard_count,
+ }
+ end
+
+ items
end
def paid
@@ -166,10 +250,10 @@ struct Video
end
def genre_url : String?
- info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
- def vr? : Bool?
+ def is_vr : Bool?
return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
end
@@ -250,21 +334,6 @@ struct Video
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # Macro to generate ? and = accessor methods for attributes in `info`
- private macro predicate_bool(method_name, name)
- # Return {{name.stringify}} from `info`
- def {{method_name.id.underscore}}? : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
-
- # Update {{name.stringify}} into `info`
- def {{method_name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
-
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
- end
-
# Method definitions, using the macros above
getset_string author
@@ -286,12 +355,11 @@ struct Video
getset_i64 likes
getset_i64 views
- # TODO: Make predicate_bool the default as to adhere to Crystal conventions
getset_bool allowRatings
getset_bool authorVerified
getset_bool isFamilyFriendly
getset_bool isListed
- predicate_bool upcoming, isUpcoming
+ getset_bool isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -326,6 +394,10 @@ end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
+ allowed_regions = info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
+
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index c811cfe1..484e61d2 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -123,7 +123,6 @@ module Invidious::Videos
"Esperanto",
"Estonian",
"Filipino",
- "Filipino (auto-generated)",
"Finnish",
"French",
"French (auto-generated)",
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
index 1371bebb..c7191dec 100644
--- a/src/invidious/videos/description.cr
+++ b/src/invidious/videos/description.cr
@@ -36,13 +36,7 @@ def parse_description(desc, video_id : String) : String?
return "" if content.empty?
commands = desc["commandRuns"]?.try &.as_a
- if commands.nil?
- # Slightly faster than HTML.escape, as we're only doing one pass on
- # the string instead of five for the standard library
- return String.build do |str|
- copy_string(str, content.each_codepoint, content.size)
- end
- end
+ return content if commands.nil?
# Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
# (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index feb58440..0e1a947c 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -36,13 +36,6 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
- if published_time_text = related["publishedTimeText"]?
- decoded_time = decode_date(published_time_text["simpleText"].to_s)
- published = decoded_time.to_rfc3339.to_s
- else
- published = nil
- end
-
# TODO: when refactoring video types, make a struct for related videos
# or reuse an existing type, if that fits.
return {
@@ -54,7 +47,6 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
"view_count" => JSON::Any.new(view_count || "0"),
"short_view_count" => JSON::Any.new(short_view_count || "0"),
"author_verified" => JSON::Any.new(author_verified),
- "published" => JSON::Any.new(published || ""),
}
end
@@ -63,7 +55,7 @@ def extract_video_info(video_id : String)
client_config = YoutubeAPI::ClientConfig.new
# Fetch data from the player endpoint
- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
@@ -82,7 +74,7 @@ def extract_video_info(video_id : String)
"reason" => JSON::Any.new(reason),
}
end
- elsif video_id != player_response.dig?("videoDetails", "videoId")
+ elsif video_id != player_response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
# Line to be reverted if one day we solve the video not available issue.
@@ -108,51 +100,37 @@ def extract_video_info(video_id : String)
params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason
- if !CONFIG.invidious_companion.present?
- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
- players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
+ new_player_response = nil
- players_fallback.each do |player_fallback|
- client_config.client_type = player_fallback
-
- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
-
- if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
- streaming_data = player_response["streamingData"].as_h
- streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
- player_response["streamingData"] = JSON::Any.new(streaming_data)
- break
- end
- rescue InfoException
- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
- end
- end
-
- # Seems like video page can still render even without playable streams.
- # its better than nothing.
- #
- # # Were we able to find playable video streams?
- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- # # No :(
- # end
+ if reason.nil?
+ # Fetch the video streams using an Android client in order to get the
+ # decrypted URLs and maybe fix throttling issues (#2194). See the
+ # following issue for an explanation about decrypted URLs:
+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+ client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+ new_player_response = try_fetch_streaming_data(video_id, client_config)
end
- {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+ # Last hope
+ if new_player_response.nil?
+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
+ new_player_response = try_fetch_streaming_data(video_id, client_config)
+ end
+
+ # Replace player response and reset reason
+ if !new_player_response.nil?
+ # Preserve captions & storyboard data before replacement
+ new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+ new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+
+ player_response = new_player_response
+ params.delete("reason")
+ end
+
+ {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
params[f] = player_response[f] if player_response[f]?
end
- # Convert URLs, if those are present
- if streaming_data = player_response["streamingData"]?
- %w[formats adaptiveFormats].each do |key|
- streaming_data.as_h[key]?.try &.as_a.each do |format|
- format.as_h["url"] = JSON::Any.new(convert_url(format))
- end
- end
-
- params["streamingData"] = streaming_data
- end
-
# Data structure version, for cache control
params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
@@ -166,7 +144,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
- if id != response.dig?("videoDetails", "videoId")
+ if id != response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise InfoException.new(
@@ -202,11 +180,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
end
video_details = player_response.dig?("videoDetails")
- if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
- microformat = {} of String => JSON::Any
- end
+ microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
raise BrokenTubeException.new("videoDetails") if !video_details
+ raise BrokenTubeException.new("microformat") if !microformat
# Basic video infos
@@ -231,17 +208,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
- premiere_timestamp ||= player_response.dig?(
- "playabilityStatus", "liveStreamability",
- "liveStreamabilityRenderer", "offlineSlate",
- "liveStreamOfflineSlateRenderer", "scheduledStartTime"
- )
- .try &.as_s.to_i64
- .try { |t| Time.unix(t) }
-
live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
- .try &.as_bool
- live_now ||= video_details.dig?("isLive").try &.as_bool || false
+ .try &.as_bool || false
post_live_dvr = video_details.dig?("isPostLiveDvr")
.try &.as_bool || false
@@ -252,7 +220,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
.try &.as_a.map &.as_s || [] of String
allow_ratings = video_details["allowRatings"]?.try &.as_bool
- family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+ family_friendly = microformat["isFamilySafe"].try &.as_bool
is_listed = video_details["isCrawlable"]?.try &.as_bool
is_upcoming = video_details["isUpcoming"]?.try &.as_bool
@@ -456,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
"shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
# Video metadata
"genre" => JSON::Any.new(genre.try &.as_s || ""),
- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
"license" => JSON::Any.new(license.try &.as_s || ""),
# Music section
"music" => JSON.parse(music_list.to_json),
@@ -470,35 +438,3 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
return params
end
-
-private def convert_url(fmt)
- if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
- sp = cfr["sp"]
- url = URI.parse(cfr["url"])
- params = url.query_params
-
- LOGGER.debug("convert_url: Decoding '#{cfr}'")
-
- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
- params[sp] = unsig if unsig
- else
- url = URI.parse(fmt["url"].as_s)
- params = url.query_params
- end
-
- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
- params["n"] = n if n
-
- if token = CONFIG.po_token
- params["pot"] = token
- end
-
- url.query_params = params
- LOGGER.trace("convert_url: new url is '#{url}'")
-
- return url.to_s
-rescue ex
- LOGGER.debug("convert_url: Error when parsing video URL")
- LOGGER.trace(ex.inspect_with_backtrace)
- return ""
-end
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
deleted file mode 100644
index bd0eef59..00000000
--- a/src/invidious/videos/storyboard.cr
+++ /dev/null
@@ -1,122 +0,0 @@
-require "uri"
-require "http/params"
-
-module Invidious::Videos
- struct Storyboard
- # Template URL
- getter url : URI
- getter proxied_url : URI
-
- # Thumbnail parameters
- getter width : Int32
- getter height : Int32
- getter count : Int32
- getter interval : Int32
-
- # Image (storyboard) parameters
- getter rows : Int32
- getter columns : Int32
- getter images_count : Int32
-
- def initialize(
- *, @url, @width, @height, @count, @interval,
- @rows, @columns, @images_count,
- )
- authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
-
- @proxied_url = URI.parse(HOST_URL)
- @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
- @proxied_url.query = @url.query
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
- # Livestream storyboards are a bit different
- # TODO: document exactly how
- if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [Storyboard.new(
- url: URI.parse(storyboard.split("#")[0]),
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- rows: 3,
- columns: 3,
- images_count: -1
- )]
- end
-
- # Split the storyboard string into chunks
- #
- # General format (whitespaces added for legibility):
- # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
- # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
- # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
- # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
- #
- storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- return [] of Storyboard if !storyboards
-
- # The base URL is the first chunk
- base_url = URI.parse(storyboards.shift)
-
- return storyboards.map_with_index do |sb, i|
- # Separate the different storyboard parameters:
- # width/height: respective dimensions, in pixels, of a single thumbnail
- # count: how many thumbnails are displayed across the full video
- # columns/rows: maximum amount of thumbnails that can be stuffed in a
- # single image, horizontally and vertically.
- # interval: interval between two thumbnails, in milliseconds
- # name: storyboard filename. Usually "M$M" or "default"
- # sigh: URL cryptographic signature
- width, height, count, columns, rows, interval, name, sigh = sb.split("#")
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- columns = columns.to_i
- rows = rows.to_i
-
- # Copy base URL object, so that we can modify it
- url = base_url.dup
-
- # Add the signature to the URL
- params = url.query_params
- params["sigh"] = sigh
- url.query_params = params
-
- # Replace the template parts with what we have
- url.path = url.path.sub("$L", i).sub("$N", name)
-
- # This value represents the maximum amount of thumbnails that can fit
- # in a single image. The last image (or the only one for short videos)
- # will contain less thumbnails than that.
- thumbnails_per_image = columns * rows
-
- # This value represents the total amount of storyboards required to
- # hold all of the thumbnails. It can't be less than 1.
- images_count = (count / thumbnails_per_image).ceil.to_i
-
- # Compute the interval when needed (in general, that's only required
- # for the first "default" storyboard).
- if interval == 0
- interval = ((length_seconds / count) * 1_000).to_i
- end
-
- Storyboard.new(
- url: url,
- width: width,
- height: height,
- count: count,
- interval: interval,
- rows: rows,
- columns: columns,
- images_count: images_count,
- )
- end
- end
- end
-end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index ee1272d1..dac00eea 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -1,26 +1,8 @@
module Invidious::Videos
- # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
- # These lines can be categorized into two types: section headings and regular lines representing content from the video.
- struct Transcript
- # Types
- record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- alias TranscriptLine = HeadingLine | RegularLine
+ # Namespace for methods primarily relating to Transcripts
+ module Transcript
+ record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- property lines : Array(TranscriptLine)
-
- property language_code : String
- property auto_generated : Bool
-
- # User friendly label for the current transcript.
- # Example: "English (auto-generated)"
- property label : String
-
- # Initializes a new Transcript struct with the contents and associated metadata describing it
- def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
- end
-
- # Generates a protobuf string to fetch the requested transcript from YouTube
def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
kind = auto_generated ? "asr" : ""
@@ -48,114 +30,48 @@ module Invidious::Videos
return params
end
- # Constructs a Transcripts struct from the initial YouTube response
- def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
- transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
- "content", "transcriptSearchPanelRenderer")
+ def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
+ # Convert into array of TranscriptLine
+ lines = self.parse(initial_data)
- segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
-
- if !segment_list["initialSegments"]?
- raise NotFoundException.new("Requested transcript does not exist")
- end
-
- # Extract user-friendly label for the current transcript
-
- footer_language_menu = transcript_panel.dig?(
- "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
- )
-
- if footer_language_menu
- label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
- else
- label = language_code
- end
-
- # Extract transcript lines
-
- initial_segments = segment_list["initialSegments"].as_a
-
- lines = [] of TranscriptLine
-
- initial_segments.each do |line|
- if unpacked_line = line["transcriptSectionHeaderRenderer"]?
- line_type = HeadingLine
- else
- unpacked_line = line["transcriptSegmentRenderer"]
- line_type = RegularLine
- end
-
- start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
- end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
- text = extract_text(unpacked_line["snippet"]) || ""
-
- lines << line_type.new(start_ms, end_ms, text)
- end
-
- return Transcript.new(
- lines: lines,
- language_code: language_code,
- auto_generated: auto_generated,
- label: label
- )
- end
-
- # Converts transcript lines to a WebVTT file
- #
- # This is used within Invidious to replace subtitles
- # as to workaround YouTube's rate-limited timedtext endpoint.
- def to_vtt
settings_field = {
"Kind" => "captions",
- "Language" => @language_code,
+ "Language" => target_language,
}
- vtt = WebVTT.build(settings_field) do |builder|
- @lines.each do |line|
- # Section headers are excluded from the VTT conversion as to
- # match the regular captions returned from YouTube as much as possible
- next if line.is_a? HeadingLine
-
- builder.cue(line.start_ms, line.end_ms, line.line)
+ # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
+ vtt = WebVTT.build(settings_field) do |vtt|
+ lines.each do |line|
+ vtt.cue(line.start_ms, line.end_ms, line.line)
end
end
return vtt
end
- def to_json(json : JSON::Builder)
- json.field "languageCode", @language_code
- json.field "autoGenerated", @auto_generated
- json.field "label", @label
- json.field "body" do
- json.array do
- @lines.each do |line|
- json.object do
- if line.is_a? HeadingLine
- json.field "type", "heading"
- else
- json.field "type", "regular"
- end
+ private def self.parse(initial_data : Hash(String, JSON::Any))
+ body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+ "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
+ "initialSegments").as_a
- json.field "startMs", line.start_ms.total_milliseconds
- json.field "endMs", line.end_ms.total_milliseconds
- json.field "line", line.line
- end
- end
+ lines = [] of TranscriptLine
+ body.each do |line|
+ # Transcript section headers. They are not apart of the captions and as such we can safely skip them.
+ if line.as_h.has_key?("transcriptSectionHeaderRenderer")
+ next
end
- end
- end
- def to_json
- JSON.build do |json|
- json.object do
- json.field "transcript" do
- json.object do
- to_json(json)
- end
- end
- end
+ line = line["transcriptSegmentRenderer"]
+
+ start_ms = line["startMs"].as_s.to_i.millisecond
+ end_ms = line["endMs"].as_s.to_i.millisecond
+
+ text = extract_text(line["snippet"]) || ""
+
+ lines << TranscriptLine.new(start_ms, end_ms, text)
end
+
+ return lines
end
end
end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
index 48177bd8..34cf7ff0 100644
--- a/src/invidious/videos/video_preferences.cr
+++ b/src/invidious/videos/video_preferences.cr
@@ -2,7 +2,6 @@ struct VideoPreferences
include JSON::Serializable
property annotations : Bool
- property preload : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
@@ -29,7 +28,6 @@ end
def process_video_params(query, preferences)
annotations = query["iv_load_policy"]?.try &.to_i?
- preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
comments = query["comments"]?.try &.split(",").map(&.downcase)
continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -52,7 +50,6 @@ def process_video_params(query, preferences)
if preferences
# region ||= preferences.region
annotations ||= preferences.annotations.to_unsafe
- preload ||= preferences.preload.to_unsafe
autoplay ||= preferences.autoplay.to_unsafe
comments ||= preferences.comments
continue ||= preferences.continue.to_unsafe
@@ -73,7 +70,6 @@ def process_video_params(query, preferences)
end
annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- preload ||= CONFIG.default_user_preferences.preload.to_unsafe
autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
comments ||= CONFIG.default_user_preferences.comments
continue ||= CONFIG.default_user_preferences.continue.to_unsafe
@@ -93,7 +89,6 @@ def process_video_params(query, preferences)
save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
annotations = annotations == 1
- preload = preload == 1
autoplay = autoplay == 1
continue = continue == 1
continue_autoplay = continue_autoplay == 1
@@ -133,7 +128,6 @@ def process_video_params(query, preferences)
params = VideoPreferences.new({
annotations: annotations,
- preload: preload,
autoplay: autoplay,
comments: comments,
continue: continue,
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 686de6bd..09df106d 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -11,7 +11,6 @@
when .channels? then "/channel/#{ucid}/channels"
when .podcasts? then "/channel/#{ucid}/podcasts"
when .releases? then "/channel/#{ucid}/releases"
- when .courses? then "/channel/#{ucid}/courses"
else
"/channel/#{ucid}"
end
@@ -21,9 +20,7 @@
page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
base_url: relative_url,
- ctoken: next_continuation,
- first_page: continuation.nil?,
- params: env.params.query,
+ ctoken: next_continuation
)
%>
@@ -33,18 +30,16 @@
-
+
-
+
<%- end -%>
-
-
<%= author %> - Invidious
<% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 132e636c..d2a305d3 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -7,7 +7,7 @@
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
- selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
-%>
<% content_for "header" do %>
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index a24423df..6d227cfc 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -1,6 +1,6 @@
<%-
thin_mode = env.get("preferences").as(Preferences).thin_mode
- item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+ item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil
author_verified = item.responds_to?(:author_verified) && item.author_verified
-%>
@@ -97,18 +97,6 @@
<% when Category %>
- <% when ProblematicTimelineItem %>
-
-
-
- <%=translate(locale, "timeline_parse_error_placeholder_heading")%>
- <%=translate(locale, "timeline_parse_error_placeholder_message")%>
-
-
- <%=translate(locale, "timeline_parse_error_show_technical_details")%>
- <%=get_issue_template(env, item.parse_exception)[1]%>
-
-
<% else %>
<%-
# `endpoint_params` is used for the "video-context-buttons" component
@@ -140,7 +128,7 @@
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7c584d15..955e0855 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -46,14 +46,8 @@ struct PlaylistVideo XML.build { |xml| to_xml(xml) } end - def to_json(locale : String?, json : JSON::Builder) - to_json(json) - end - def to_json(json : JSON::Builder, index : Int32? = nil) json.object do - json.field "type", "video" - json.field "title", self.title json.field "videoId", self.id @@ -73,7 +67,6 @@ struct PlaylistVideo end json.field "lengthSeconds", self.length_seconds - json.field "liveNow", self.live_now end end @@ -270,7 +263,7 @@ end def subscribe_playlist(user, playlist) playlist = InvidiousPlaylist.new({ - title: playlist.title[..150], + title: playlist.title.byte_slice(0, 150), id: playlist.id, author: user.email, description: "", # Max 5000 characters @@ -373,8 +366,6 @@ def fetch_playlist(plid : String) if text.includes? "video" video_count = text.gsub(/\D/, "").to_i? || 0 - elsif text.includes? "episode" - video_count = text.gsub(/\D/, "").to_i? || 0 elsif text.includes? "view" views = text.gsub(/\D/, "").to_i64? || 0_i64 else @@ -432,7 +423,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -448,7 +439,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -500,14 +491,12 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) index: index, }) end - rescue ex - videos << ProblematicTimelineItem.new(parse_exception: ex) end return videos end -def template_playlist(playlist, listen) +def template_playlist(playlist) html = <<-END_HTML
@@ -521,7 +510,7 @@ def template_playlist(playlist, listen)
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
-
+
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr
index c8db207c..5b55738e 100644
--- a/src/invidious/routes/account.cr
+++ b/src/invidious/routes/account.cr
@@ -53,7 +53,7 @@ module Invidious::Routes::Account
return error_template(401, "Password is a required field")
end
- new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
+ new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
@@ -123,10 +123,8 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- view_name = "subscriptions_#{sha256(user.email)}"
Invidious::Database::Users.delete(user)
Invidious::Database::SessionIDs.delete(email: user.email)
- PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}")
env.request.cookies.each do |cookie|
cookie.expires = Time.utc(1990, 1, 1)
@@ -240,7 +238,7 @@ module Invidious::Routes::Account
return error_template(400, ex)
end
- scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
+ scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
@@ -328,9 +326,17 @@ module Invidious::Routes::Account
end
end
- case action = env.params.query["action"]?
- when "revoke_token"
- session = env.params.query["session"]
+ if env.params.query["action_revoke_token"]?
+ action = "action_revoke_token"
+ else
+ return env.redirect referer
+ end
+
+ session = env.params.query["session"]?
+ session ||= ""
+
+ case action
+ when .starts_with? "action_revoke_token"
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr
index c27caad7..d89e752c 100644
--- a/src/invidious/routes/api/manifest.cr
+++ b/src/invidious/routes/api/manifest.cr
@@ -8,11 +8,6 @@ module Invidious::Routes::API::Manifest
id = env.params.url["id"]
region = env.params.query["region"]?
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
- end
-
# Since some implementations create playlists based on resolution regardless of different codecs,
# we can opt to only add a source to a representation if it has a unique height within that representation
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -32,21 +27,28 @@ module Invidious::Routes::API::Manifest
haltf env, status_code: response.status_code
end
- # Proxy URLs for video playback on invidious.
- # Other API clients can get the original URLs by omiting `local=true`.
- manifest = response.body.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
- url = baseurl.lchop("").rchop(" ")
- url = HttpServer::Utils.proxy_video_url(url, absolute: true) if local
+ manifest = response.body
+
+ manifest = manifest.gsub(/[^<]+<\/BaseURL>/) do |baseurl|
+ url = baseurl.lchop("")
+ url = url.rchop(" ")
+
+ if local
+ uri = URI.parse(url)
+ url = "#{HOST_URL}#{uri.request_target}host/#{uri.host}/"
+ end
+
"#{url} "
end
return manifest
end
- # Ditto, only proxify URLs if `local=true` is used
+ adaptive_fmts = video.adaptive_fmts
+
if local
- video.adaptive_fmts.each do |fmt|
- fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s, absolute: true))
+ adaptive_fmts.each do |fmt|
+ fmt["url"] = JSON::Any.new("#{HOST_URL}#{URI.parse(fmt["url"].as_s).request_target}")
end
end
@@ -68,23 +70,17 @@ module Invidious::Routes::API::Manifest
# OTF streams aren't supported yet (See https://github.com/TeamNewPipe/NewPipe/issues/2415)
next if !(fmt.has_key?("indexRange") && fmt.has_key?("initRange"))
- audio_track = fmt["audioTrack"]?.try &.as_h? || {} of String => JSON::Any
- lang = audio_track["id"]?.try &.as_s.split('.')[0] || "und"
- is_default = audio_track.has_key?("audioIsDefault") ? audio_track["audioIsDefault"].as_bool : i == 0
- displayname = audio_track["displayName"]?.try &.as_s || "Unknown"
- bitrate = fmt["bitrate"]
-
# Different representations of the same audio should be groupped into one AdaptationSet.
# However, most players don't support auto quality switching, so we have to trick them
# into providing a quality selector.
# See https://github.com/iv-org/invidious/issues/3074 for more details.
- xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: "#{displayname} [#{bitrate}k]", lang: lang) do
+ xml.element("AdaptationSet", id: i, mimeType: mime_type, startWithSAP: 1, subsegmentAlignment: true, label: fmt["bitrate"].to_s + "k") do
codecs = fmt["mimeType"].as_s.split("codecs=")[1].strip('"')
bandwidth = fmt["bitrate"].as_i
itag = fmt["itag"].as_i
url = fmt["url"].as_s
- xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: is_default ? "main" : "alternate")
+ xml.element("Role", schemeIdUri: "urn:mpeg:dash:role:2011", value: i == 0 ? "main" : "alternate")
xml.element("Representation", id: fmt["itag"], codecs: codecs, bandwidth: bandwidth) do
xml.element("AudioChannelConfiguration", schemeIdUri: "urn:mpeg:dash:23003:3:audio_channel_configuration:2011",
@@ -181,9 +177,8 @@ module Invidious::Routes::API::Manifest
manifest = response.body
if local
- manifest = manifest.gsub(/https:\/\/[^\n"]*/m) do |match|
- uri = URI.parse(match)
- path = uri.path
+ manifest = manifest.gsub(/^https:\/\/\w+---.{11}\.c\.youtube\.com[^\n]*/m) do |match|
+ path = URI.parse(match).path
path = path.lchop("/videoplayback/")
path = path.rchop("/")
@@ -212,7 +207,7 @@ module Invidious::Routes::API::Manifest
raw_params["fvip"] = fvip["fvip"]
end
- raw_params["host"] = uri.host.not_nil!
+ raw_params["local"] = "true"
"#{HOST_URL}/videoplayback?#{raw_params}"
end
diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr
index a940ee68..7faf200a 100644
--- a/src/invidious/routes/api/v1/channels.cr
+++ b/src/invidious/routes/api/v1/channels.cr
@@ -27,21 +27,10 @@ module Invidious::Routes::API::V1::Channels
# Retrieve "sort by" setting from URL parameters
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+ rescue ex
+ return error_json(500, ex)
end
JSON.build do |json|
@@ -95,7 +84,6 @@ module Invidious::Routes::API::V1::Channels
json.field "joined", channel.joined.to_unix
json.field "autoGenerated", channel.auto_generated
- json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
@@ -154,23 +142,12 @@ module Invidious::Routes::API::V1::Channels
sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_videos(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_videos(
+ channel, continuation: continuation, sort_by: sort_by
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -197,26 +174,14 @@ module Invidious::Routes::API::V1::Channels
get_channel()
# Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_shorts(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_shorts(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -243,26 +208,14 @@ module Invidious::Routes::API::V1::Channels
get_channel()
# Retrieve continuation from URL parameters
- sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
continuation = env.params.query["continuation"]?
- if channel.is_age_gated
- begin
- playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
- videos = get_playlist_videos(playlist, offset: 0)
- rescue ex : InfoException
- # playlist doesnt exist.
- videos = [] of PlaylistVideo
- end
- next_continuation = nil
- else
- begin
- videos, next_continuation = Channel::Tabs.get_60_livestreams(
- channel, continuation: continuation, sort_by: sort_by
- )
- rescue ex
- return error_json(500, ex)
- end
+ begin
+ videos, next_continuation = Channel::Tabs.get_60_livestreams(
+ channel, continuation: continuation
+ )
+ rescue ex
+ return error_json(500, ex)
end
return JSON.build do |json|
@@ -368,35 +321,6 @@ module Invidious::Routes::API::V1::Channels
end
end
- def self.courses(env)
- locale = env.get("preferences").as(Preferences).locale
-
- env.response.content_type = "application/json"
-
- ucid = env.params.url["ucid"]
- continuation = env.params.query["continuation"]?
-
- # Use the macro defined above
- channel = nil # Make the compiler happy
- get_channel()
-
- items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
-
- JSON.build do |json|
- json.object do
- json.field "playlists" do
- json.array do
- items.each do |item|
- item.to_json(locale, json) if item.is_a?(SearchPlaylist)
- end
- end
- end
-
- json.field "continuation", next_continuation if next_continuation
- end
- end
- end
-
def self.community(env)
locale = env.get("preferences").as(Preferences).locale
diff --git a/src/invidious/routes/api/v1/feeds.cr b/src/invidious/routes/api/v1/feeds.cr
index fea2993c..41865f34 100644
--- a/src/invidious/routes/api/v1/feeds.cr
+++ b/src/invidious/routes/api/v1/feeds.cr
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
if !CONFIG.popular_enabled
error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
- haltf env, 403, error_message
+ haltf env, 400, error_message
end
JSON.build do |json|
diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr
index 4f5b58da..12942906 100644
--- a/src/invidious/routes/api/v1/misc.cr
+++ b/src/invidious/routes/api/v1/misc.cr
@@ -42,9 +42,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
if plid.starts_with? "RD"
return env.redirect "/api/v1/mixes/#{plid}"
end
@@ -77,9 +74,7 @@ module Invidious::Routes::API::V1::Misc
response = playlist.to_json(offset, video_id: video_id)
json_response = JSON.parse(response)
- if json_response["videos"].as_a.empty?
- json_response = JSON.parse(response)
- elsif json_response["videos"].as_a[0]["index"] != offset
+ if json_response["videos"].as_a[0]["index"] != offset
offset = json_response["videos"].as_a[0]["index"].as_i
lookback = offset < 50 ? offset : 50
response = playlist.to_json(offset - lookback)
@@ -88,7 +83,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
- playlist_html = template_playlist(json_response, listen)
+ playlist_html = template_playlist(json_response)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@@ -114,9 +109,6 @@ module Invidious::Routes::API::V1::Misc
format = env.params.query["format"]?
format ||= "json"
- listen_param = env.params.query["listen"]?
- listen = (listen_param == "true" || listen_param == "1")
-
begin
mix = fetch_mix(rdid, continuation, locale: locale)
@@ -147,7 +139,9 @@ module Invidious::Routes::API::V1::Misc
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "videoThumbnails" do
- Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ json.array do
+ Invidious::JSONify::APIv1.thumbnails(json, video.id)
+ end
end
json.field "index", video.index
@@ -161,7 +155,7 @@ module Invidious::Routes::API::V1::Misc
if format == "html"
response = JSON.parse(response)
- playlist_html = template_mix(response, listen)
+ playlist_html = template_mix(response)
next_video = response["videos"].as_a.select { |video| !video["author"].as_s.empty? }[0]?.try &.["videoId"]
response = {
@@ -183,8 +177,8 @@ module Invidious::Routes::API::V1::Misc
begin
resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"]
- page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
- if page_type == "WEB_PAGE_TYPE_UNKNOWN"
+ pageType = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
+ if pageType == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url")
end
@@ -200,7 +194,7 @@ module Invidious::Routes::API::V1::Misc
json.field "playlistId", sub_endpoint["playlistId"].as_s if sub_endpoint["playlistId"]?
json.field "startTimeSeconds", sub_endpoint["startTimeSeconds"].as_i if sub_endpoint["startTimeSeconds"]?
json.field "params", params.try &.as_s
- json.field "pageType", page_type
+ json.field "pageType", pageType
end
end
end
diff --git a/src/invidious/routes/api/v1/search.cr b/src/invidious/routes/api/v1/search.cr
index 59a30745..2922b060 100644
--- a/src/invidious/routes/api/v1/search.cr
+++ b/src/invidious/routes/api/v1/search.cr
@@ -31,7 +31,9 @@ module Invidious::Routes::API::V1::Search
query = env.params.query["q"]? || ""
begin
- client = make_client(URI.parse("https://suggestqueries-clients6.youtube.com"), force_youtube_headers: true)
+ client = HTTP::Client.new("suggestqueries-clients6.youtube.com")
+ client.before_request { |r| add_yt_headers(r) }
+
url = "/complete/search?client=youtube&hl=en&gl=#{region}&q=#{URI.encode_www_form(query)}&gs_ri=youtube&ds=yt"
response = client.get(url).body
diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr
index 6a3eb8ae..9281f4dd 100644
--- a/src/invidious/routes/api/v1/videos.cr
+++ b/src/invidious/routes/api/v1/videos.cr
@@ -1,5 +1,3 @@
-require "html"
-
module Invidious::Routes::API::V1::Videos
def self.videos(env)
locale = env.get("preferences").as(Preferences).locale
@@ -91,14 +89,9 @@ module Invidious::Routes::API::V1::Videos
if CONFIG.use_innertube_for_captions
params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+ initial_data = YoutubeAPI.get_transcript(params)
- transcript = Invidious::Videos::Transcript.from_raw(
- YoutubeAPI.get_transcript(params),
- caption.language_code,
- caption.auto_generated
- )
-
- webvtt = transcript.to_vtt
+ webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
else
# Timedtext API handling
url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
@@ -118,7 +111,7 @@ module Invidious::Routes::API::V1::Videos
else
caption_xml = XML.parse(caption_xml)
- webvtt = WebVTT.build(settings_field) do |builder|
+ webvtt = WebVTT.build(settings_field) do |webvtt|
caption_nodes = caption_xml.xpath_nodes("//transcript/text")
caption_nodes.each_with_index do |node, i|
start_time = node["start"].to_f.seconds
@@ -138,16 +131,12 @@ module Invidious::Routes::API::V1::Videos
text = "#{md["text"]} "
end
- builder.cue(start_time, end_time, text)
+ webvtt.cue(start_time, end_time, text)
end
end
end
else
- uri = URI.parse(url)
- query_params = uri.query_params
- query_params["fmt"] = "vtt"
- uri.query_params = query_params
- webvtt = YT_POOL.client &.get(uri.request_target).body
+ webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
if webvtt.starts_with?(" 0
url += "?#{env.params.query}"
@@ -74,15 +72,13 @@ module Invidious::Routes::Embed
url = "/playlist?list=#{plid}"
raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url))
end
-
- first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex
return error_template(500, ex)
end
- url = "/embed/#{first_playlist_video.id}"
+ url = "/embed/#{videos[0].id}"
elsif video_series
url = "/embed/#{video_series.shift}"
env.params.query["playlist"] = video_series.join(",")
@@ -161,12 +157,10 @@ module Invidious::Routes::Embed
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -207,14 +201,6 @@ module Invidious::Routes::Embed
return env.redirect url
end
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
rendered "embed"
end
end
diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr
index 070c96eb..e20a7139 100644
--- a/src/invidious/routes/feeds.cr
+++ b/src/invidious/routes/feeds.cr
@@ -143,25 +143,32 @@ module Invidious::Routes::Feeds
# RSS feeds
def self.rss_channel(env)
+ locale = env.get("preferences").as(Preferences).locale
+
env.response.headers["Content-Type"] = "application/atom+xml"
env.response.content_type = "application/atom+xml"
- if env.params.url["ucid"].matches?(/^[\w-]+$/)
- ucid = env.params.url["ucid"]
- else
- return error_atom(400, InfoException.new("Invalid channel ucid provided."))
- end
+ ucid = env.params.url["ucid"]
params = HTTP::Params.parse(env.params.query["params"]? || "")
+ begin
+ channel = get_about_info(ucid, locale)
+ rescue ex : ChannelRedirect
+ return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
+ rescue ex : NotFoundException
+ return error_atom(404, ex)
+ rescue ex
+ return error_atom(500, ex)
+ end
+
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
- response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
- return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
+ response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}")
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@@ -172,7 +179,7 @@ module Invidious::Routes::Feeds
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
- video_ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+ ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
description_html = entry.xpath_node("media:group/media:description", namespaces).not_nil!.to_s
views = entry.xpath_node("media:group/media:community/media:statistics", namespaces).not_nil!.["views"].to_i64
@@ -180,44 +187,43 @@ module Invidious::Routes::Feeds
title: title,
id: video_id,
author: author,
- ucid: video_ucid,
+ ucid: ucid,
published: published,
views: views,
description_html: description_html,
length_seconds: 0,
+ live_now: false,
+ paid: false,
+ premium: false,
premiere_timestamp: nil,
author_verified: false,
- author_thumbnail: nil,
- badges: VideoBadges::None,
})
end
- author = ""
- author = videos[0].author if videos.size > 0
-
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
"xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom",
"xml:lang": "en-US") do
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
- xml.element("id") { xml.text "yt:channel:#{ucid}" }
- xml.element("yt:channelId") { xml.text ucid }
- xml.element("title") { xml.text author }
- xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{ucid}")
+ xml.element("id") { xml.text "yt:channel:#{channel.ucid}" }
+ xml.element("yt:channelId") { xml.text channel.ucid }
+ xml.element("icon") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
+ xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}")
xml.element("author") do
- xml.element("name") { xml.text author }
- xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
+ xml.element("name") { xml.text channel.author }
+ xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" }
end
xml.element("image") do
- xml.element("url") { xml.text "" }
- xml.element("title") { xml.text author }
+ xml.element("url") { xml.text channel.author_thumbnail }
+ xml.element("title") { xml.text channel.author }
xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}")
end
videos.each do |video|
- video.to_xml(false, params, xml)
+ video.to_xml(channel.auto_generated, params, xml)
end
end
end
@@ -296,13 +302,7 @@ module Invidious::Routes::Feeds
xml.element("name") { xml.text playlist.author }
end
- videos.each do |video|
- if video.is_a? PlaylistVideo
- video.to_xml(xml)
- else
- video.to_xml(env, locale, xml)
- end
- end
+ videos.each &.to_xml(xml)
end
end
else
@@ -311,9 +311,8 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
- return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
-
document = XML.parse(response.body)
+
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|
node.attributes.each do |attribute|
case attribute.name
@@ -426,6 +425,16 @@ module Invidious::Routes::Feeds
next # skip this video since it raised an exception (e.g. it is a scheduled live event)
end
+ if CONFIG.enable_user_notifications
+ # Deliver notifications to `/api/v1/auth/notifications`
+ payload = {
+ "topic" => video.ucid,
+ "videoId" => video.id,
+ "published" => published.to_unix,
+ }.to_json
+ PG_DB.exec("NOTIFY notifications, E'#{payload}'")
+ end
+
video = ChannelVideo.new({
id: id,
title: video.title,
@@ -441,7 +450,11 @@ module Invidious::Routes::Feeds
was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
if was_insert
- NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
+ if CONFIG.enable_user_notifications
+ Invidious::Database::Users.add_notification(video)
+ else
+ Invidious::Database::Users.feed_needs_update(video)
+ end
end
end
end
diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr
index 51d85dfe..b6a2e110 100644
--- a/src/invidious/routes/images.cr
+++ b/src/invidious/routes/images.cr
@@ -11,9 +11,29 @@ module Invidious::Routes::Images
end
end
+ # We're encapsulating this into a proc in order to easily reuse this
+ # portion of the code for each request block below.
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ env.response.headers.delete("Transfer-Encoding")
+ return
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- GGPHT_POOL.client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -41,10 +61,27 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Connection"] = "close"
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool(authority).client &.get(url, headers) do |resp|
- env.response.headers["Connection"] = "close"
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -64,9 +101,26 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool("i9").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
@@ -111,7 +165,8 @@ module Invidious::Routes::Images
if name == "maxres.jpg"
build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
- if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
+ # This can likely be optimized into a (small) pool sometime in the future.
+ if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg"
break
end
@@ -126,28 +181,29 @@ module Invidious::Routes::Images
end
end
+ request_proc = ->(response : HTTP::Client::Response) {
+ env.response.status_code = response.status_code
+ response.headers.each do |key, value|
+ if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
+ env.response.headers[key] = value
+ end
+ end
+
+ env.response.headers["Access-Control-Allow-Origin"] = "*"
+
+ if response.status_code >= 300 && response.status_code != 404
+ return env.response.headers.delete("Transfer-Encoding")
+ end
+
+ proxy_file(response, env)
+ }
+
begin
- get_ytimg_pool("i").client &.get(url, headers) do |resp|
- return self.proxy_image(env, resp)
+ # This can likely be optimized into a (small) pool sometime in the future.
+ HTTP::Client.get("https://i.ytimg.com#{url}") do |resp|
+ return request_proc.call(resp)
end
rescue ex
end
end
-
- private def self.proxy_image(env, response)
- env.response.status_code = response.status_code
- response.headers.each do |key, value|
- if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
- env.response.headers[key] = value
- end
- end
-
- env.response.headers["Access-Control-Allow-Origin"] = "*"
-
- if response.status_code >= 300
- return env.response.headers.delete("Transfer-Encoding")
- end
-
- return proxy_file(response, env)
- end
end
diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr
index e7de5018..add9f75d 100644
--- a/src/invidious/routes/login.cr
+++ b/src/invidious/routes/login.cr
@@ -21,6 +21,9 @@ module Invidious::Routes::Login
account_type = env.params.query["type"]?
account_type ||= "invidious"
+ captcha_type = env.params.query["captcha"]?
+ captcha_type ||= "image"
+
templated "user/login"
end
@@ -85,14 +88,34 @@ module Invidious::Routes::Login
password = password.byte_slice(0, 55)
if CONFIG.captcha_enabled
+ captcha_type = env.params.body["captcha_type"]?
answer = env.params.body["answer"]?
+ change_type = env.params.body["change_type"]?
- account_type = "invidious"
- captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ if !captcha_type || change_type
+ if change_type
+ captcha_type = change_type
+ end
+ captcha_type ||= "image"
+
+ account_type = "invidious"
+
+ if captcha_type == "image"
+ captcha = Invidious::User::Captcha.generate_image(HMAC_KEY)
+ else
+ captcha = Invidious::User::Captcha.generate_text(HMAC_KEY)
+ end
+
+ return templated "user/login"
+ end
tokens = env.params.body.select { |k, _| k.match(/^token\[\d+\]$/) }.map { |_, v| v }
- if answer
+ answer ||= ""
+ captcha_type ||= "image"
+
+ case captcha_type
+ when "image"
answer = answer.lstrip('0')
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
@@ -101,8 +124,27 @@ module Invidious::Routes::Login
rescue ex
return error_template(400, ex)
end
- else
- return templated "user/login"
+ else # "text"
+ answer = Digest::MD5.hexdigest(answer.downcase.strip)
+
+ if tokens.empty?
+ return error_template(500, "Erroneous CAPTCHA")
+ end
+
+ found_valid_captcha = false
+ error_exception = Exception.new
+ tokens.each do |tok|
+ begin
+ validate_request(tok, answer, env.request, HMAC_KEY, locale)
+ found_valid_captcha = true
+ rescue ex
+ error_exception = ex
+ end
+ end
+
+ if !found_valid_captcha
+ return error_template(500, error_exception)
+ end
end
end
@@ -118,9 +160,6 @@ module Invidious::Routes::Login
Invidious::Database::Users.insert(user)
Invidious::Database::SessionIDs.insert(sid, email)
- view_name = "subscriptions_#{sha256(user.email)}"
- PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
-
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
if env.request.cookies["PREFS"]?
diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr
index 0b868755..d6bd9571 100644
--- a/src/invidious/routes/misc.cr
+++ b/src/invidious/routes/misc.cr
@@ -40,21 +40,7 @@ module Invidious::Routes::Misc
def self.cross_instance_redirect(env)
referer = get_referer(env)
-
- instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"]
- # Filter out the current instance
- other_available_instances = instance_list.reject { |_, domain| domain == CONFIG.domain }
-
- if other_available_instances.empty?
- # If the current instance is the only one, use the redirect URL as fallback
- instance_url = "redirect.invidious.io"
- else
- # Select other random instance
- # Sample returns an array
- # Instances are packaged as {region, domain} in the instance list
- instance_url = other_available_instances.sample(1)[0][1]
- end
-
+ instance_url = fetch_random_instance
env.redirect "https://#{instance_url}#{referer}"
end
end
diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr
index f2213da4..9c6843e9 100644
--- a/src/invidious/routes/playlists.cr
+++ b/src/invidious/routes/playlists.cr
@@ -304,6 +304,23 @@ module Invidious::Routes::Playlists
end
end
+ if env.params.query["action_create_playlist"]?
+ action = "action_create_playlist"
+ elsif env.params.query["action_delete_playlist"]?
+ action = "action_delete_playlist"
+ elsif env.params.query["action_edit_playlist"]?
+ action = "action_edit_playlist"
+ elsif env.params.query["action_add_video"]?
+ action = "action_add_video"
+ video_id = env.params.query["video_id"]
+ elsif env.params.query["action_remove_video"]?
+ action = "action_remove_video"
+ elsif env.params.query["action_move_video_before"]?
+ action = "action_move_video_before"
+ else
+ return env.redirect referer
+ end
+
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
@@ -318,8 +335,12 @@ module Invidious::Routes::Playlists
end
end
- case action = env.params.query["action"]?
- when "add_video"
+ email = user.email
+
+ case action
+ when "action_edit_playlist"
+ # TODO: Playlist stub
+ when "action_add_video"
if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
@@ -356,14 +377,12 @@ module Invidious::Routes::Playlists
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
- when "remove_video"
+ when "action_remove_video"
index = env.params.query["set_video_id"]
Invidious::Database::PlaylistVideos.delete(index)
Invidious::Database::Playlists.update_video_removed(playlist_id, index)
- when "move_video_before"
+ when "action_move_video_before"
# TODO: Playlist stub
- when nil
- return error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
end
diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr
index 39ca77c0..112535bd 100644
--- a/src/invidious/routes/preferences.cr
+++ b/src/invidious/routes/preferences.cr
@@ -27,10 +27,6 @@ module Invidious::Routes::PreferencesRoute
annotations_subscribed ||= "off"
annotations_subscribed = annotations_subscribed == "on"
- preload = env.params.body["preload"]?.try &.as(String)
- preload ||= "off"
- preload = preload == "on"
-
autoplay = env.params.body["autoplay"]?.try &.as(String)
autoplay ||= "off"
autoplay = autoplay == "on"
@@ -148,7 +144,6 @@ module Invidious::Routes::PreferencesRoute
preferences = Preferences.from_json({
annotations: annotations,
annotations_subscribed: annotations_subscribed,
- preload: preload,
autoplay: autoplay,
captions: captions,
comments: comments,
@@ -219,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
statistics_enabled ||= "off"
CONFIG.statistics_enabled = statistics_enabled == "on"
- CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
+ CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
File.write("config/config.yml", CONFIG.to_yaml)
end
diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr
index b195c7b3..5be33533 100644
--- a/src/invidious/routes/search.cr
+++ b/src/invidious/routes/search.cr
@@ -51,18 +51,8 @@ module Invidious::Routes::Search
else
user = env.get? "user"
- # An URL was copy/pasted in the search box.
- # Redirect the user to the appropriate page.
- if query.url?
- return env.redirect UrlSanitizer.process(query.text).to_s
- end
-
begin
- if user
- items = query.process(user.as(User))
- else
- items = query.process
- end
+ items = query.process
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
diff --git a/src/invidious/routes/subscriptions.cr b/src/invidious/routes/subscriptions.cr
index 1de655d2..7f9ec592 100644
--- a/src/invidious/routes/subscriptions.cr
+++ b/src/invidious/routes/subscriptions.cr
@@ -32,16 +32,24 @@ module Invidious::Routes::Subscriptions
end
end
+ if env.params.query["action_create_subscription_to_channel"]?.try &.to_i?.try &.== 1
+ action = "action_create_subscription_to_channel"
+ elsif env.params.query["action_remove_subscriptions"]?.try &.to_i?.try &.== 1
+ action = "action_remove_subscriptions"
+ else
+ return env.redirect referer
+ end
+
channel_id = env.params.query["c"]?
channel_id ||= ""
- case action = env.params.query["action"]?
- when "create_subscription_to_channel"
+ case action
+ when "action_create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
- when "remove_subscriptions"
+ when "action_remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr
index 083087a9..ec18f3b8 100644
--- a/src/invidious/routes/video_playback.cr
+++ b/src/invidious/routes/video_playback.cr
@@ -21,7 +21,7 @@ module Invidious::Routes::VideoPlayback
end
# Sanity check, to avoid being used as an open proxy
- if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
+ if !host.matches?(/[\w-]+.googlevideo.com/)
return error_template(400, "Invalid \"host\" parameter.")
end
@@ -37,13 +37,12 @@ module Invidious::Routes::VideoPlayback
# See: https://github.com/iv-org/invidious/issues/3302
range_header = env.request.headers["Range"]?
- sq = query_params["sq"]?
- if range_header.nil? && sq.nil?
+ if range_header.nil?
range_for_head = query_params["range"]? || "0-640"
headers["Range"] = "bytes=#{range_for_head}"
end
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
response = HTTP::Client::Response.new(500)
error = ""
5.times do
@@ -58,7 +57,7 @@ module Invidious::Routes::VideoPlayback
if new_host != host
host = new_host
client.close
- client = make_client(URI.parse(new_host), region, force_resolve: true)
+ client = make_client(URI.parse(new_host), region, force_resolve = true)
end
url = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
@@ -72,7 +71,7 @@ module Invidious::Routes::VideoPlayback
fvip = "3"
host = "https://r#{fvip}---#{mn}.googlevideo.com"
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
rescue ex
error = ex.message
end
@@ -132,7 +131,7 @@ module Invidious::Routes::VideoPlayback
end
# TODO: Record bytes written so we can restart after a chunk fails
- loop do
+ while true
if !range_end && content_length
range_end = content_length
end
@@ -165,13 +164,10 @@ module Invidious::Routes::VideoPlayback
env.response.headers["Access-Control-Allow-Origin"] = "*"
if location = resp.headers["Location"]?
- url = Invidious::HttpServer::Utils.proxy_video_url(location, region: region)
+ location = URI.parse(location)
+ location = "#{location.request_target}&host=#{location.host}#{region ? "®ion=#{region}" : ""}"
- if title = query_params["title"]?
- url = "#{url}&title=#{URI.encode_www_form(title)}"
- end
-
- env.redirect url
+ env.redirect location
break
end
@@ -200,7 +196,7 @@ module Invidious::Routes::VideoPlayback
break
else
client.close
- client = make_client(URI.parse(host), region, force_resolve: true)
+ client = make_client(URI.parse(host), region, force_resolve = true)
end
end
@@ -257,11 +253,6 @@ module Invidious::Routes::VideoPlayback
# YouTube /videoplayback links expire after 6 hours,
# so we have a mechanism here to redirect to the latest version
def self.latest_version(env)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
- end
-
id = env.params.query["id"]?
itag = env.params.query["itag"]?.try &.to_i?
diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr
index e777b3f1..aabe8dfc 100644
--- a/src/invidious/routes/watch.cr
+++ b/src/invidious/routes/watch.cr
@@ -121,12 +121,10 @@ module Invidious::Routes::Watch
adaptive_fmts = video.adaptive_fmts
if params.local
- fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
+ fmt_stream.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
+ adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(URI.parse(fmt["url"].as_s).request_target) }
end
- # Always proxy DASH streams, otherwise youtube CORS headers will prevent playback
- adaptive_fmts.each { |fmt| fmt["url"] = JSON::Any.new(HttpServer::Utils.proxy_video_url(fmt["url"].as_s)) }
-
video_streams = video.video_streams
audio_streams = video.audio_streams
@@ -192,14 +190,6 @@ module Invidious::Routes::Watch
captions: video.captions
)
- if CONFIG.invidious_companion.present?
- invidious_companion = CONFIG.invidious_companion.sample
- env.response.headers["Content-Security-Policy"] =
- env.response.headers["Content-Security-Policy"]
- .gsub("media-src", "media-src #{invidious_companion.public_url}")
- .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
- end
-
templated "watch"
end
@@ -251,10 +241,18 @@ module Invidious::Routes::Watch
end
end
- case action = env.params.query["action"]?
- when "mark_watched"
+ if env.params.query["action_mark_watched"]?
+ action = "action_mark_watched"
+ elsif env.params.query["action_mark_unwatched"]?
+ action = "action_mark_unwatched"
+ else
+ return env.redirect referer
+ end
+
+ case action
+ when "action_mark_watched"
Invidious::Database::Users.mark_watched(user, id)
- when "mark_unwatched"
+ when "action_mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
@@ -293,9 +291,6 @@ module Invidious::Routes::Watch
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
end
- if CONFIG.invidious_companion.present?
- return error_template(403, "Downloads should be routed through Companion when present")
- end
title = env.params.body["title"]? || ""
video_id = env.params.body["id"]? || ""
@@ -325,9 +320,10 @@ module Invidious::Routes::Watch
env.params.query["label"] = URI.decode_www_form(label.as_s)
return Invidious::Routes::API::V1::Videos.captions(env)
- elsif itag = download_widget["itag"]?.try &.as_i.to_s
+ elsif itag = download_widget["itag"]?.try &.as_i
# URL params specific to /latest_version
env.params.query["id"] = video_id
+ env.params.query["itag"] = itag.to_s
env.params.query["title"] = filename
env.params.query["local"] = "true"
diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr
index 46b71f1f..ba05da19 100644
--- a/src/invidious/routing.cr
+++ b/src/invidious/routing.cr
@@ -120,10 +120,8 @@ module Invidious::Routing
get "/channel/:ucid/streams", Routes::Channels, :streams
get "/channel/:ucid/podcasts", Routes::Channels, :podcasts
get "/channel/:ucid/releases", Routes::Channels, :releases
- get "/channel/:ucid/courses", Routes::Channels, :courses
get "/channel/:ucid/playlists", Routes::Channels, :playlists
get "/channel/:ucid/community", Routes::Channels, :community
- get "/channel/:ucid/posts", Routes::Channels, :community
get "/channel/:ucid/channels", Routes::Channels, :channels
get "/channel/:ucid/about", Routes::Channels, :about
@@ -238,7 +236,6 @@ module Invidious::Routing
get "/api/v1/annotations/:id", {{namespace}}::Videos, :annotations
get "/api/v1/comments/:id", {{namespace}}::Videos, :comments
get "/api/v1/clips/:id", {{namespace}}::Videos, :clips
- get "/api/v1/transcripts/:id", {{namespace}}::Videos, :transcripts
# Feeds
get "/api/v1/trending", {{namespace}}::Feeds, :trending
@@ -246,18 +243,17 @@ module Invidious::Routing
# Channels
get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
- get "/api/v1/channels/:ucid/latest", {{namespace}}::Channels, :latest
- get "/api/v1/channels/:ucid/videos", {{namespace}}::Channels, :videos
get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
get "/api/v1/channels/:ucid/podcasts", {{namespace}}::Channels, :podcasts
get "/api/v1/channels/:ucid/releases", {{namespace}}::Channels, :releases
- get "/api/v1/channels/:ucid/courses", {{namespace}}::Channels, :courses
- get "/api/v1/channels/:ucid/playlists", {{namespace}}::Channels, :playlists
- get "/api/v1/channels/:ucid/community", {{namespace}}::Channels, :community
- get "/api/v1/channels/:ucid/posts", {{namespace}}::Channels, :community
+
get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
- get "/api/v1/channels/:ucid/search", {{namespace}}::Channels, :search
+
+ {% for route in {"videos", "latest", "playlists", "community", "search"} %}
+ get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
+ get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}
+ {% end %}
# Posts
get "/api/v1/post/:id", {{namespace}}::Channels, :post
@@ -275,6 +271,11 @@ module Invidious::Routing
# Authenticated
+ # The notification APIs cannot be extracted yet! They require the *local* notifications constant defined in invidious.cr
+ #
+ # Invidious::Routing.get "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+ # Invidious::Routing.post "/api/v1/auth/notifications", {{namespace}}::Authenticated, :notifications
+
get "/api/v1/auth/preferences", {{namespace}}::Authenticated, :get_preferences
post "/api/v1/auth/preferences", {{namespace}}::Authenticated, :set_preferences
diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr
index bc2715cf..bf968734 100644
--- a/src/invidious/search/filters.cr
+++ b/src/invidious/search/filters.cr
@@ -75,7 +75,7 @@ module Invidious::Search
@type : Type = Type::All,
@duration : Duration = Duration::None,
@features : Features = Features::None,
- @sort : Sort = Sort::Relevance,
+ @sort : Sort = Sort::Relevance
)
end
diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr
index 25edb936..10b81c59 100644
--- a/src/invidious/search/processors.cr
+++ b/src/invidious/search/processors.cr
@@ -37,18 +37,18 @@ module Invidious::Search
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
- view_name = "subscriptions_#{sha256(user.email)}"
-
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
- SELECT *,
- to_tsvector(#{view_name}.title) ||
- to_tsvector(#{view_name}.author)
- as document
- FROM #{view_name}
- ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
- query.text, (query.page - 1) * 20,
+ SELECT cv.*,
+ to_tsvector(cv.title) ||
+ to_tsvector(cv.author) AS document
+ FROM channel_videos cv
+ JOIN users ON cv.ucid = any(users.subscriptions)
+ WHERE users.email = $1 AND published > now() - interval '1 month'
+ ORDER BY published
+ ) v_search WHERE v_search.document @@ plainto_tsquery($2) LIMIT 20 OFFSET $3;",
+ user.email, query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
diff --git a/src/invidious/search/query.cr b/src/invidious/search/query.cr
index 94a92e23..e38845d9 100644
--- a/src/invidious/search/query.cr
+++ b/src/invidious/search/query.cr
@@ -20,9 +20,6 @@ module Invidious::Search
property region : String?
property channel : String = ""
- # Flag that indicates if the smart search features have been disabled.
- @inhibit_ssf : Bool = false
-
# Return true if @raw_query is either `nil` or empty
private def empty_raw_query?
return @raw_query.empty?
@@ -47,22 +44,14 @@ module Invidious::Search
def initialize(
params : HTTP::Params,
@type : Type = Type::Regular,
- @region : String? = nil,
+ @region : String? = nil
)
# Get the raw search query string (common to all search types). In
# Regular search mode, also look for the `search_query` URL parameter
- _raw_query = params["q"]?
- _raw_query ||= params["search_query"]? if @type.regular?
- _raw_query ||= ""
-
- # Remove surrounding whitespaces. Mostly useful for copy/pasted URLs.
- @raw_query = _raw_query.strip
-
- # Check for smart features (ex: URL search) inhibitor (backslash).
- # If inhibitor is present, remove it.
- if @raw_query.starts_with?('\\')
- @inhibit_ssf = true
- @raw_query = @raw_query[1..]
+ if @type.regular?
+ @raw_query = params["q"]? || params["search_query"]? || ""
+ else
+ @raw_query = params["q"]? || ""
end
# Get the page number (also common to all search types)
@@ -96,7 +85,7 @@ module Invidious::Search
@filters = Filters.from_iv_params(params)
@channel = params["channel"]? || ""
- if @filters.default? && @raw_query.index(/\w:\w/)
+ if @filters.default? && @raw_query.includes?(':')
# Parse legacy filters from query
@filters, @channel, @query, subs = Filters.from_legacy_filters(@raw_query)
else
@@ -147,22 +136,5 @@ module Invidious::Search
return params
end
-
- # Checks if the query is a standalone URL
- def url? : Bool
- # If the smart features have been inhibited, don't go further.
- return false if @inhibit_ssf
-
- # Only supported in regular search mode
- return false if !@type.regular?
-
- # If filters are present, that's a regular search
- return false if !@filters.default?
-
- # Simple heuristics: domain name
- return @raw_query.starts_with?(
- /(https?:\/\/)?(www\.)?(m\.)?youtu(\.be|be\.com)\//
- )
- end
end
end
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr
index d14cde5d..107d148d 100644
--- a/src/invidious/trending.cr
+++ b/src/invidious/trending.cr
@@ -31,12 +31,12 @@ def fetch_trending(trending_type, region, locale)
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
- extracted.concat itm.contents.select(SearchItem)
+ extracted.concat extract_category(itm)
else
extracted << itm
end
end
# Deduplicate items before returning results
- return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
+ return extracted.select(SearchVideo).uniq!(&.id), plid
end
diff --git a/src/invidious/user/captcha.cr b/src/invidious/user/captcha.cr
index b175c3b9..8a0f67e5 100644
--- a/src/invidious/user/captcha.cr
+++ b/src/invidious/user/captcha.cr
@@ -4,6 +4,8 @@ struct Invidious::User
module Captcha
extend self
+ private TEXTCAPTCHA_URL = URI.parse("https://textcaptcha.com")
+
def generate_image(key)
second = Random::Secure.rand(12)
second_angle = second * 30
@@ -58,5 +60,19 @@ struct Invidious::User
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
+
+ def generate_text(key)
+ response = make_client(TEXTCAPTCHA_URL, &.get("/github.com/iv.org/invidious.json").body)
+ response = JSON.parse(response)
+
+ tokens = response["a"].as_a.map do |answer|
+ generate_response(answer.as_s, {":login"}, key, use_nonce: true)
+ end
+
+ return {
+ question: response["q"].as_s,
+ tokens: tokens,
+ }
+ end
end
end
diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr
index 007eb666..108f2ccc 100644
--- a/src/invidious/user/imports.cr
+++ b/src/invidious/user/imports.cr
@@ -115,7 +115,7 @@ struct Invidious::User
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
- privacy = item["privacy"]?.try &.as_s?.try { |raw_pl_privacy_state| PlaylistPrivacy.parse? raw_pl_privacy_state }
+ privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
@@ -124,7 +124,7 @@ struct Invidious::User
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
- item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
+ videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
if idx > CONFIG.playlist_length_limit
raise InfoException.new("Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
@@ -161,7 +161,7 @@ struct Invidious::User
# Youtube
# -------------------
- private def opml?(mimetype : String, extension : String)
+ private def is_opml?(mimetype : String, extension : String)
opml_mimetypes = [
"application/xml",
"text/xml",
@@ -179,10 +179,10 @@ struct Invidious::User
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
- if opml?(type, extension)
+ if is_opml?(type, extension)
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
- channel["xmlUrl"].match!(/UC[a-zA-Z0-9_-]{22}/)[0]
+ channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
@@ -290,39 +290,42 @@ struct Invidious::User
end
def from_newpipe(user : User, body : String) : Bool
- Compress::Zip::File.open(IO::Memory.new(body), true) do |file|
- entry = file.entries.find { |file_entry| file_entry.filename == "newpipe.db" }
- return false if entry.nil?
- entry.open do |file_io|
- # Ensure max size of 4MB
- io_sized = IO::Sized.new(file_io, 0x400000)
+ io = IO::Memory.new(body)
- begin
- temp = File.tempfile(".db") do |tempfile|
- begin
- File.write(tempfile.path, io_sized.gets_to_end)
- rescue
- return false
- end
+ Compress::Zip::File.open(io) do |file|
+ file.entries.each do |entry|
+ entry.open do |file_io|
+ # Ensure max size of 4MB
+ io_sized = IO::Sized.new(file_io, 0x400000)
- DB.open("sqlite3://" + tempfile.path) do |db|
- user.watched += db.query_all("SELECT url FROM streams", as: String)
- .map(&.lchop("https://www.youtube.com/watch?v="))
+ next if entry.filename != "newpipe.db"
- user.watched.uniq!
- Invidious::Database::Users.update_watch_history(user)
+ tempfile = File.tempfile(".db")
- user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
- .map(&.lchop("https://www.youtube.com/channel/"))
-
- user.subscriptions.uniq!
- user.subscriptions = get_batch_channels(user.subscriptions)
-
- Invidious::Database::Users.update_subscriptions(user)
- end
+ begin
+ File.write(tempfile.path, io_sized.gets_to_end)
+ rescue
+ return false
end
- ensure
- temp.delete if !temp.nil?
+
+ db = DB.open("sqlite3://" + tempfile.path)
+
+ user.watched += db.query_all("SELECT url FROM streams", as: String)
+ .map(&.lchop("https://www.youtube.com/watch?v="))
+
+ user.watched.uniq!
+ Invidious::Database::Users.update_watch_history(user)
+
+ user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
+ .map(&.lchop("https://www.youtube.com/channel/"))
+
+ user.subscriptions.uniq!
+ user.subscriptions = get_batch_channels(user.subscriptions)
+
+ Invidious::Database::Users.update_subscriptions(user)
+
+ db.close
+ tempfile.delete
end
end
end
diff --git a/src/invidious/user/preferences.cr b/src/invidious/user/preferences.cr
index 0a8525f3..b3059403 100644
--- a/src/invidious/user/preferences.cr
+++ b/src/invidious/user/preferences.cr
@@ -4,7 +4,6 @@ struct Preferences
property annotations : Bool = CONFIG.default_user_preferences.annotations
property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
- property preload : Bool = CONFIG.default_user_preferences.preload
property autoplay : Bool = CONFIG.default_user_preferences.autoplay
property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
diff --git a/src/invidious/users.cr b/src/invidious/users.cr
index 65566d20..0b2d1ef5 100644
--- a/src/invidious/users.cr
+++ b/src/invidious/users.cr
@@ -27,7 +27,6 @@ def get_subscription_feed(user, max_results = 40, page = 1)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
- view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
@@ -53,33 +52,39 @@ def get_subscription_feed(user, max_results = 40, page = 1)
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
+ # "SELECT cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = $1 AND published > now() - interval '1 month' ORDER BY published DESC"
+ # "SELECT DISTINCT ON (cv.ucid) cv.* FROM channel_videos cv JOIN users ON cv.ucid = any(users.subscriptions) WHERE users.email = ? AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' ORDER BY ucid, published DESC"
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
else
# Show latest video from each channel
- videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT DISTINCT ON (cv.ucid) cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY ucid, published DESC", user.email, as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
-
- if user.watched.empty?
- values = "'{}'"
- else
- values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
- end
- videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND NOT cv.id = any(users.watched) AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
-
- videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+ videos = PG_DB.query_all("SELECT cv.* " \
+ "FROM channel_videos cv " \
+ "JOIN users ON cv.ucid = any(users.subscriptions) " \
+ "WHERE users.email = $1 AND published > now() - interval '1 month' " \
+ "ORDER BY published DESC LIMIT $2 OFFSET $3", user.email, limit, offset, as: ChannelVideo)
end
end
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index 348a0a66..c218b4ef 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -15,7 +15,7 @@ struct Video
# NOTE: don't forget to bump this number if any change is made to
# the `params` structure in videos/parser.cr!!!
#
- SCHEMA_VERSION = 3
+ SCHEMA_VERSION = 2
property id : String
@@ -26,6 +26,12 @@ struct Video
@[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Captions::Metadata
+ @[DB::Field(ignore: true)]
+ property adaptive_fmts : Array(Hash(String, JSON::Any))?
+
+ @[DB::Field(ignore: true)]
+ property fmt_stream : Array(Hash(String, JSON::Any))?
+
@[DB::Field(ignore: true)]
property description : String?
@@ -92,24 +98,45 @@ struct Video
# Methods for parsing streaming data
- def fmt_stream : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "formats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
+
+ fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @fmt_stream = fmt_stream
+ return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
- def adaptive_fmts : Array(Hash(String, JSON::Any))
- if formats = info.dig?("streamingData", "adaptiveFormats")
- return formats
- .as_a.map(&.as_h)
- .sort_by! { |f| f["width"]?.try &.as_i || f["audioTrack"]?.try { |a| a["audioIsDefault"]?.try { |v| v.as_bool ? -1 : 0 } } || 0 }
- else
- return [] of Hash(String, JSON::Any)
+ def adaptive_fmts
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
+ fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
+ fmt_stream.each do |fmt|
+ if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
+ s.each do |k, v|
+ fmt[k] = JSON::Any.new(v)
+ end
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
+ end
+
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
+ fmt["url"] = JSON::Any.new("#{fmt["url"]}®ion=#{self.info["region"]}") if self.info["region"]?
end
+
+ fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
+ @adaptive_fmts = fmt_stream
+ return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
@@ -123,8 +150,65 @@ struct Video
# Misc. methods
def storyboards
- container = info.dig?("storyboards") || JSON::Any.new("{}")
- return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
+ storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
+ .try &.as_s.split("|")
+
+ if !storyboards
+ if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+ return [{
+ url: storyboard.split("#")[0],
+ width: 106,
+ height: 60,
+ count: -1,
+ interval: 5000,
+ storyboard_width: 3,
+ storyboard_height: 3,
+ storyboard_count: -1,
+ }]
+ end
+ end
+
+ items = [] of NamedTuple(
+ url: String,
+ width: Int32,
+ height: Int32,
+ count: Int32,
+ interval: Int32,
+ storyboard_width: Int32,
+ storyboard_height: Int32,
+ storyboard_count: Int32)
+
+ return items if !storyboards
+
+ url = URI.parse(storyboards.shift)
+ params = HTTP::Params.parse(url.query || "")
+
+ storyboards.each_with_index do |sb, i|
+ width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
+ params["sigh"] = sigh
+ url.query = params.to_s
+
+ width = width.to_i
+ height = height.to_i
+ count = count.to_i
+ interval = interval.to_i
+ storyboard_width = storyboard_width.to_i
+ storyboard_height = storyboard_height.to_i
+ storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
+
+ items << {
+ url: url.to_s.sub("$L", i).sub("$N", "M$M"),
+ width: width,
+ height: height,
+ count: count,
+ interval: interval,
+ storyboard_width: storyboard_width,
+ storyboard_height: storyboard_height,
+ storyboard_count: storyboard_count,
+ }
+ end
+
+ items
end
def paid
@@ -166,10 +250,10 @@ struct Video
end
def genre_url : String?
- info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
+ info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
- def vr? : Bool?
+ def is_vr : Bool?
return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
end
@@ -250,21 +334,6 @@ struct Video
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
- # Macro to generate ? and = accessor methods for attributes in `info`
- private macro predicate_bool(method_name, name)
- # Return {{name.stringify}} from `info`
- def {{method_name.id.underscore}}? : Bool
- return info[{{name.stringify}}]?.try &.as_bool || false
- end
-
- # Update {{name.stringify}} into `info`
- def {{method_name.id.underscore}}=(value : Bool)
- info[{{name.stringify}}] = JSON::Any.new(value)
- end
-
- {% if flag?(:debug_macros) %} {{debug}} {% end %}
- end
-
# Method definitions, using the macros above
getset_string author
@@ -286,12 +355,11 @@ struct Video
getset_i64 likes
getset_i64 views
- # TODO: Make predicate_bool the default as to adhere to Crystal conventions
getset_bool allowRatings
getset_bool authorVerified
getset_bool isFamilyFriendly
getset_bool isListed
- predicate_bool upcoming, isUpcoming
+ getset_bool isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
@@ -326,6 +394,10 @@ end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
+ allowed_regions = info
+ .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
+ .try &.as_a.map &.as_s || [] of String
+
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr
index c811cfe1..484e61d2 100644
--- a/src/invidious/videos/caption.cr
+++ b/src/invidious/videos/caption.cr
@@ -123,7 +123,6 @@ module Invidious::Videos
"Esperanto",
"Estonian",
"Filipino",
- "Filipino (auto-generated)",
"Finnish",
"French",
"French (auto-generated)",
diff --git a/src/invidious/videos/description.cr b/src/invidious/videos/description.cr
index 1371bebb..c7191dec 100644
--- a/src/invidious/videos/description.cr
+++ b/src/invidious/videos/description.cr
@@ -36,13 +36,7 @@ def parse_description(desc, video_id : String) : String?
return "" if content.empty?
commands = desc["commandRuns"]?.try &.as_a
- if commands.nil?
- # Slightly faster than HTML.escape, as we're only doing one pass on
- # the string instead of five for the standard library
- return String.build do |str|
- copy_string(str, content.each_codepoint, content.size)
- end
- end
+ return content if commands.nil?
# Not everything is stored in UTF-8 on youtube's side. The SMP codepoints
# (0x10000 and above) are encoded as UTF-16 surrogate pairs, which are
diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr
index feb58440..0e1a947c 100644
--- a/src/invidious/videos/parser.cr
+++ b/src/invidious/videos/parser.cr
@@ -36,13 +36,6 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
- if published_time_text = related["publishedTimeText"]?
- decoded_time = decode_date(published_time_text["simpleText"].to_s)
- published = decoded_time.to_rfc3339.to_s
- else
- published = nil
- end
-
# TODO: when refactoring video types, make a struct for related videos
# or reuse an existing type, if that fits.
return {
@@ -54,7 +47,6 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
"view_count" => JSON::Any.new(view_count || "0"),
"short_view_count" => JSON::Any.new(short_view_count || "0"),
"author_verified" => JSON::Any.new(author_verified),
- "published" => JSON::Any.new(published || ""),
}
end
@@ -63,7 +55,7 @@ def extract_video_info(video_id : String)
client_config = YoutubeAPI::ClientConfig.new
# Fetch data from the player endpoint
- player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config)
+ player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
@@ -82,7 +74,7 @@ def extract_video_info(video_id : String)
"reason" => JSON::Any.new(reason),
}
end
- elsif video_id != player_response.dig?("videoDetails", "videoId")
+ elsif video_id != player_response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
# Line to be reverted if one day we solve the video not available issue.
@@ -108,51 +100,37 @@ def extract_video_info(video_id : String)
params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason
- if !CONFIG.invidious_companion.present?
- if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
- players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
+ new_player_response = nil
- players_fallback.each do |player_fallback|
- client_config.client_type = player_fallback
-
- next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
-
- if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
- streaming_data = player_response["streamingData"].as_h
- streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
- player_response["streamingData"] = JSON::Any.new(streaming_data)
- break
- end
- rescue InfoException
- next LOGGER.warn("Failed to fetch streams with #{player_fallback}")
- end
- end
-
- # Seems like video page can still render even without playable streams.
- # its better than nothing.
- #
- # # Were we able to find playable video streams?
- # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
- # # No :(
- # end
+ if reason.nil?
+ # Fetch the video streams using an Android client in order to get the
+ # decrypted URLs and maybe fix throttling issues (#2194). See the
+ # following issue for an explanation about decrypted URLs:
+ # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+ client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+ new_player_response = try_fetch_streaming_data(video_id, client_config)
end
- {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+ # Last hope
+ if new_player_response.nil?
+ client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
+ new_player_response = try_fetch_streaming_data(video_id, client_config)
+ end
+
+ # Replace player response and reset reason
+ if !new_player_response.nil?
+ # Preserve captions & storyboard data before replacement
+ new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+ new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+
+ player_response = new_player_response
+ params.delete("reason")
+ end
+
+ {"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
params[f] = player_response[f] if player_response[f]?
end
- # Convert URLs, if those are present
- if streaming_data = player_response["streamingData"]?
- %w[formats adaptiveFormats].each do |key|
- streaming_data.as_h[key]?.try &.as_a.each do |format|
- format.as_h["url"] = JSON::Any.new(convert_url(format))
- end
- end
-
- params["streamingData"] = streaming_data
- end
-
# Data structure version, for cache control
params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
@@ -166,7 +144,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
- if id != response.dig?("videoDetails", "videoId")
+ if id != response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise InfoException.new(
@@ -202,11 +180,10 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
end
video_details = player_response.dig?("videoDetails")
- if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
- microformat = {} of String => JSON::Any
- end
+ microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
raise BrokenTubeException.new("videoDetails") if !video_details
+ raise BrokenTubeException.new("microformat") if !microformat
# Basic video infos
@@ -231,17 +208,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
- premiere_timestamp ||= player_response.dig?(
- "playabilityStatus", "liveStreamability",
- "liveStreamabilityRenderer", "offlineSlate",
- "liveStreamOfflineSlateRenderer", "scheduledStartTime"
- )
- .try &.as_s.to_i64
- .try { |t| Time.unix(t) }
-
live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
- .try &.as_bool
- live_now ||= video_details.dig?("isLive").try &.as_bool || false
+ .try &.as_bool || false
post_live_dvr = video_details.dig?("isPostLiveDvr")
.try &.as_bool || false
@@ -252,7 +220,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
.try &.as_a.map &.as_s || [] of String
allow_ratings = video_details["allowRatings"]?.try &.as_bool
- family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+ family_friendly = microformat["isFamilySafe"].try &.as_bool
is_listed = video_details["isCrawlable"]?.try &.as_bool
is_upcoming = video_details["isUpcoming"]?.try &.as_bool
@@ -456,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
"shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
# Video metadata
"genre" => JSON::Any.new(genre.try &.as_s || ""),
- "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
+ "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
"license" => JSON::Any.new(license.try &.as_s || ""),
# Music section
"music" => JSON.parse(music_list.to_json),
@@ -470,35 +438,3 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
return params
end
-
-private def convert_url(fmt)
- if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
- sp = cfr["sp"]
- url = URI.parse(cfr["url"])
- params = url.query_params
-
- LOGGER.debug("convert_url: Decoding '#{cfr}'")
-
- unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
- params[sp] = unsig if unsig
- else
- url = URI.parse(fmt["url"].as_s)
- params = url.query_params
- end
-
- n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
- params["n"] = n if n
-
- if token = CONFIG.po_token
- params["pot"] = token
- end
-
- url.query_params = params
- LOGGER.trace("convert_url: new url is '#{url}'")
-
- return url.to_s
-rescue ex
- LOGGER.debug("convert_url: Error when parsing video URL")
- LOGGER.trace(ex.inspect_with_backtrace)
- return ""
-end
diff --git a/src/invidious/videos/storyboard.cr b/src/invidious/videos/storyboard.cr
deleted file mode 100644
index bd0eef59..00000000
--- a/src/invidious/videos/storyboard.cr
+++ /dev/null
@@ -1,122 +0,0 @@
-require "uri"
-require "http/params"
-
-module Invidious::Videos
- struct Storyboard
- # Template URL
- getter url : URI
- getter proxied_url : URI
-
- # Thumbnail parameters
- getter width : Int32
- getter height : Int32
- getter count : Int32
- getter interval : Int32
-
- # Image (storyboard) parameters
- getter rows : Int32
- getter columns : Int32
- getter images_count : Int32
-
- def initialize(
- *, @url, @width, @height, @count, @interval,
- @rows, @columns, @images_count,
- )
- authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
-
- @proxied_url = URI.parse(HOST_URL)
- @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
- @proxied_url.query = @url.query
- end
-
- # Parse the JSON structure from Youtube
- def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
- # Livestream storyboards are a bit different
- # TODO: document exactly how
- if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
- return [Storyboard.new(
- url: URI.parse(storyboard.split("#")[0]),
- width: 106,
- height: 60,
- count: -1,
- interval: 5000,
- rows: 3,
- columns: 3,
- images_count: -1
- )]
- end
-
- # Split the storyboard string into chunks
- #
- # General format (whitespaces added for legibility):
- # https://i.ytimg.com/sb//storyboard3_L$L/$N.jpg?sqp=
- # | 48 # 27 # 100 # 10 # 10 # 0 # default # rs$
- # | 80 # 45 # 95 # 10 # 10 # 10000 # M$M # rs$
- # | 160 # 90 # 95 # 5 # 5 # 10000 # M$M # rs$
- #
- storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
- .try &.as_s.split("|")
-
- return [] of Storyboard if !storyboards
-
- # The base URL is the first chunk
- base_url = URI.parse(storyboards.shift)
-
- return storyboards.map_with_index do |sb, i|
- # Separate the different storyboard parameters:
- # width/height: respective dimensions, in pixels, of a single thumbnail
- # count: how many thumbnails are displayed across the full video
- # columns/rows: maximum amount of thumbnails that can be stuffed in a
- # single image, horizontally and vertically.
- # interval: interval between two thumbnails, in milliseconds
- # name: storyboard filename. Usually "M$M" or "default"
- # sigh: URL cryptographic signature
- width, height, count, columns, rows, interval, name, sigh = sb.split("#")
-
- width = width.to_i
- height = height.to_i
- count = count.to_i
- interval = interval.to_i
- columns = columns.to_i
- rows = rows.to_i
-
- # Copy base URL object, so that we can modify it
- url = base_url.dup
-
- # Add the signature to the URL
- params = url.query_params
- params["sigh"] = sigh
- url.query_params = params
-
- # Replace the template parts with what we have
- url.path = url.path.sub("$L", i).sub("$N", name)
-
- # This value represents the maximum amount of thumbnails that can fit
- # in a single image. The last image (or the only one for short videos)
- # will contain less thumbnails than that.
- thumbnails_per_image = columns * rows
-
- # This value represents the total amount of storyboards required to
- # hold all of the thumbnails. It can't be less than 1.
- images_count = (count / thumbnails_per_image).ceil.to_i
-
- # Compute the interval when needed (in general, that's only required
- # for the first "default" storyboard).
- if interval == 0
- interval = ((length_seconds / count) * 1_000).to_i
- end
-
- Storyboard.new(
- url: url,
- width: width,
- height: height,
- count: count,
- interval: interval,
- rows: rows,
- columns: columns,
- images_count: images_count,
- )
- end
- end
- end
-end
diff --git a/src/invidious/videos/transcript.cr b/src/invidious/videos/transcript.cr
index ee1272d1..dac00eea 100644
--- a/src/invidious/videos/transcript.cr
+++ b/src/invidious/videos/transcript.cr
@@ -1,26 +1,8 @@
module Invidious::Videos
- # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
- # These lines can be categorized into two types: section headings and regular lines representing content from the video.
- struct Transcript
- # Types
- record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- alias TranscriptLine = HeadingLine | RegularLine
+ # Namespace for methods primarily relating to Transcripts
+ module Transcript
+ record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
- property lines : Array(TranscriptLine)
-
- property language_code : String
- property auto_generated : Bool
-
- # User friendly label for the current transcript.
- # Example: "English (auto-generated)"
- property label : String
-
- # Initializes a new Transcript struct with the contents and associated metadata describing it
- def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
- end
-
- # Generates a protobuf string to fetch the requested transcript from YouTube
def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
kind = auto_generated ? "asr" : ""
@@ -48,114 +30,48 @@ module Invidious::Videos
return params
end
- # Constructs a Transcripts struct from the initial YouTube response
- def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
- transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
- "content", "transcriptSearchPanelRenderer")
+ def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
+ # Convert into array of TranscriptLine
+ lines = self.parse(initial_data)
- segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
-
- if !segment_list["initialSegments"]?
- raise NotFoundException.new("Requested transcript does not exist")
- end
-
- # Extract user-friendly label for the current transcript
-
- footer_language_menu = transcript_panel.dig?(
- "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
- )
-
- if footer_language_menu
- label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
- else
- label = language_code
- end
-
- # Extract transcript lines
-
- initial_segments = segment_list["initialSegments"].as_a
-
- lines = [] of TranscriptLine
-
- initial_segments.each do |line|
- if unpacked_line = line["transcriptSectionHeaderRenderer"]?
- line_type = HeadingLine
- else
- unpacked_line = line["transcriptSegmentRenderer"]
- line_type = RegularLine
- end
-
- start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
- end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
- text = extract_text(unpacked_line["snippet"]) || ""
-
- lines << line_type.new(start_ms, end_ms, text)
- end
-
- return Transcript.new(
- lines: lines,
- language_code: language_code,
- auto_generated: auto_generated,
- label: label
- )
- end
-
- # Converts transcript lines to a WebVTT file
- #
- # This is used within Invidious to replace subtitles
- # as to workaround YouTube's rate-limited timedtext endpoint.
- def to_vtt
settings_field = {
"Kind" => "captions",
- "Language" => @language_code,
+ "Language" => target_language,
}
- vtt = WebVTT.build(settings_field) do |builder|
- @lines.each do |line|
- # Section headers are excluded from the VTT conversion as to
- # match the regular captions returned from YouTube as much as possible
- next if line.is_a? HeadingLine
-
- builder.cue(line.start_ms, line.end_ms, line.line)
+ # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
+ vtt = WebVTT.build(settings_field) do |vtt|
+ lines.each do |line|
+ vtt.cue(line.start_ms, line.end_ms, line.line)
end
end
return vtt
end
- def to_json(json : JSON::Builder)
- json.field "languageCode", @language_code
- json.field "autoGenerated", @auto_generated
- json.field "label", @label
- json.field "body" do
- json.array do
- @lines.each do |line|
- json.object do
- if line.is_a? HeadingLine
- json.field "type", "heading"
- else
- json.field "type", "regular"
- end
+ private def self.parse(initial_data : Hash(String, JSON::Any))
+ body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+ "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
+ "initialSegments").as_a
- json.field "startMs", line.start_ms.total_milliseconds
- json.field "endMs", line.end_ms.total_milliseconds
- json.field "line", line.line
- end
- end
+ lines = [] of TranscriptLine
+ body.each do |line|
+ # Transcript section headers. They are not apart of the captions and as such we can safely skip them.
+ if line.as_h.has_key?("transcriptSectionHeaderRenderer")
+ next
end
- end
- end
- def to_json
- JSON.build do |json|
- json.object do
- json.field "transcript" do
- json.object do
- to_json(json)
- end
- end
- end
+ line = line["transcriptSegmentRenderer"]
+
+ start_ms = line["startMs"].as_s.to_i.millisecond
+ end_ms = line["endMs"].as_s.to_i.millisecond
+
+ text = extract_text(line["snippet"]) || ""
+
+ lines << TranscriptLine.new(start_ms, end_ms, text)
end
+
+ return lines
end
end
end
diff --git a/src/invidious/videos/video_preferences.cr b/src/invidious/videos/video_preferences.cr
index 48177bd8..34cf7ff0 100644
--- a/src/invidious/videos/video_preferences.cr
+++ b/src/invidious/videos/video_preferences.cr
@@ -2,7 +2,6 @@ struct VideoPreferences
include JSON::Serializable
property annotations : Bool
- property preload : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
@@ -29,7 +28,6 @@ end
def process_video_params(query, preferences)
annotations = query["iv_load_policy"]?.try &.to_i?
- preload = query["preload"]?.try { |q| (q == "true" || q == "1").to_unsafe }
autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
comments = query["comments"]?.try &.split(",").map(&.downcase)
continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -52,7 +50,6 @@ def process_video_params(query, preferences)
if preferences
# region ||= preferences.region
annotations ||= preferences.annotations.to_unsafe
- preload ||= preferences.preload.to_unsafe
autoplay ||= preferences.autoplay.to_unsafe
comments ||= preferences.comments
continue ||= preferences.continue.to_unsafe
@@ -73,7 +70,6 @@ def process_video_params(query, preferences)
end
annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
- preload ||= CONFIG.default_user_preferences.preload.to_unsafe
autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
comments ||= CONFIG.default_user_preferences.comments
continue ||= CONFIG.default_user_preferences.continue.to_unsafe
@@ -93,7 +89,6 @@ def process_video_params(query, preferences)
save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
annotations = annotations == 1
- preload = preload == 1
autoplay = autoplay == 1
continue = continue == 1
continue_autoplay = continue_autoplay == 1
@@ -133,7 +128,6 @@ def process_video_params(query, preferences)
params = VideoPreferences.new({
annotations: annotations,
- preload: preload,
autoplay: autoplay,
comments: comments,
continue: continue,
diff --git a/src/invidious/views/channel.ecr b/src/invidious/views/channel.ecr
index 686de6bd..09df106d 100644
--- a/src/invidious/views/channel.ecr
+++ b/src/invidious/views/channel.ecr
@@ -11,7 +11,6 @@
when .channels? then "/channel/#{ucid}/channels"
when .podcasts? then "/channel/#{ucid}/podcasts"
when .releases? then "/channel/#{ucid}/releases"
- when .courses? then "/channel/#{ucid}/courses"
else
"/channel/#{ucid}"
end
@@ -21,9 +20,7 @@
page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
base_url: relative_url,
- ctoken: next_continuation,
- first_page: continuation.nil?,
- params: env.params.query,
+ ctoken: next_continuation
)
%>
@@ -33,18 +30,16 @@
-
+
-
+
<%- end -%>
-
-
<%= author %> - Invidious
<% end %>
diff --git a/src/invidious/views/community.ecr b/src/invidious/views/community.ecr
index 132e636c..d2a305d3 100644
--- a/src/invidious/views/community.ecr
+++ b/src/invidious/views/community.ecr
@@ -7,7 +7,7 @@
youtube_url = "https://www.youtube.com#{relative_url}"
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
- selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Posts
+ selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
-%>
<% content_for "header" do %>
diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr
index a24423df..6d227cfc 100644
--- a/src/invidious/views/components/item.ecr
+++ b/src/invidious/views/components/item.ecr
@@ -1,6 +1,6 @@
<%-
thin_mode = env.get("preferences").as(Preferences).thin_mode
- item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil
+ item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil
author_verified = item.responds_to?(:author_verified) && item.author_verified
-%>
@@ -97,18 +97,6 @@
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index c8db207c..5b55738e 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -53,7 +53,7 @@ module Invidious::Routes::Account return error_template(401, "Password is a required field") end - new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } + new_passwords = env.params.body.select { |k, v| k.match(/^new_password\[\d+\]$/) }.map { |k, v| v } if new_passwords.size <= 1 || new_passwords.uniq.size != 1 return error_template(400, "New passwords must match") @@ -123,10 +123,8 @@ module Invidious::Routes::Account return error_template(400, ex) end - view_name = "subscriptions_#{sha256(user.email)}" Invidious::Database::Users.delete(user) Invidious::Database::SessionIDs.delete(email: user.email) - PG_DB.exec("DROP MATERIALIZED VIEW #{view_name}") env.request.cookies.each do |cookie| cookie.expires = Time.utc(1990, 1, 1) @@ -240,7 +238,7 @@ module Invidious::Routes::Account return error_template(400, ex) end - scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } + scopes = env.params.body.select { |k, v| k.match(/^scopes\[\d+\]$/) }.map { |k, v| v } callback_url = env.params.body["callbackUrl"]? expire = env.params.body["expire"]?.try &.to_i? @@ -328,9 +326,17 @@ module Invidious::Routes::Account end end - case action = env.params.query["action"]? - when "revoke_token" - session = env.params.query["session"] + if env.params.query["action_revoke_token"]? + action = "action_revoke_token" + else + return env.redirect referer + end + + session = env.params.query["session"]? + session ||= "" + + case action + when .starts_with? "action_revoke_token" Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index c27caad7..d89e752c 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,11 +8,6 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? - if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" - end - # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } @@ -32,21 +27,28 @@ module Invidious::Routes::API::Manifest haltf env, status_code: response.status_code end - # Proxy URLs for video playback on invidious. - # Other API clients can get the original URLs by omiting `local=true`. - manifest = response.body.gsub(/
-
<% else %>
<%-
# `endpoint_params` is used for the "video-context-buttons" component
@@ -140,7 +128,7 @@
-
-
- <%=translate(locale, "timeline_parse_error_placeholder_heading")%>
-<%=translate(locale, "timeline_parse_error_placeholder_message")%>
-
-
- <%=translate(locale, "timeline_parse_error_show_technical_details")%>
-<%=get_issue_template(env, item.parse_exception)[1]%>-