From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Aug 2023 16:08:02 -0700 Subject: [PATCH 01/70] Add support for using HTTP proxies --- config/config.example.yml | 11 +++++++++++ shard.lock | 10 +++++++--- shard.yml | 3 +++ src/invidious.cr | 1 + src/invidious/config.cr | 11 +++++++++++ src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 38085a20..9fdc4eda 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -160,6 +160,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index efb60a59..85928c47 100644 --- a/shard.lock +++ b/shard.lock @@ -6,11 +6,11 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.1 + version: 0.1.3 backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.1 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index be06a7df..ddd510d4 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.1 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..d4114386 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b..e12054d0 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -54,6 +54,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -126,6 +135,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0e4d8aff 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,14 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn rescue ex conn.close conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -46,6 +48,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + return client end @@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 4 Oct 2023 14:36:04 -0400 Subject: [PATCH 02/70] Automatically initialize proxy via stdlib override --- .../helpers/crystal_class_overrides.cr | 34 +++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 7 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..71038703 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.10.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0e4d8aff..f34f48c5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,13 +26,15 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -48,7 +50,6 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - return client end From ccb2a6c58ef9b5b7e1c47938cfa4ec574a8560c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 28 Apr 2024 21:34:05 -0700 Subject: [PATCH 03/70] Bump http_proxy to v0.10.3 --- shard.lock | 4 ++-- shard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index 85928c47..99b4e99c 100644 --- a/shard.lock +++ b/shard.lock @@ -6,7 +6,7 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.3 + version: 0.1.1 backtracer: git: https://github.com/sija/backtracer.cr.git @@ -22,7 +22,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.10.1 + version: 0.10.3 kemal: git: https://github.com/kemalcr/kemal.git diff --git a/shard.yml b/shard.yml index ddd510d4..70ebad0a 100644 --- a/shard.yml +++ b/shard.yml @@ -30,7 +30,7 @@ dependencies: version: ~> 0.1.1 http_proxy: github: mamantoha/http_proxy - version: ~> 0.10.1 + version: ~> 0.10.3 development_dependencies: spectator: From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:10:46 -0700 Subject: [PATCH 04/70] Validate override for crystal 1.12.1 --- src/invidious/helpers/crystal_class_overrides.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index 71038703..a7d2a5e6 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -20,7 +20,7 @@ class HTTP::Client # Override stdlib to automatically initialize proxy if configured # - # Accurate as of crystal 1.10.1 + # Accurate as of crystal 1.12.1 def initialize(@host : String, port = nil, tls : TLSContext = nil) check_host_only(@host) From 288e1dccda2256a9014364d693b3eb3d7933b242 Mon Sep 17 00:00:00 2001 From: giacomocerquone Date: Thu, 13 Jun 2024 01:10:35 +0200 Subject: [PATCH 05/70] Fix player menus hiding onHover --- assets/css/player.css | 1 + 1 file changed, 1 insertion(+) diff --git a/assets/css/player.css b/assets/css/player.css index 50c7a748..9cb400ad 100644 --- a/assets/css/player.css +++ b/assets/css/player.css @@ -68,6 +68,7 @@ .video-js.player-style-youtube .vjs-menu-button-popup .vjs-menu { margin-bottom: 2em; + padding-top: 2em } .video-js.player-style-youtube .vjs-progress-control .vjs-progress-holder, .video-js.player-style-youtube .vjs-progress-control {height: 5px; From 480e073fa9be184b6839619c38795af582247c19 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:20:17 -0800 Subject: [PATCH 06/70] Use HTTP pools for image requests to YouTube --- src/invidious.cr | 8 ++++++++ src/invidious/routes/images.cr | 12 +++++------- src/invidious/yt_backend/connection_pool.cr | 15 +++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..81db2c6c 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -92,6 +92,14 @@ SOFTWARE = { YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +# Image request pool + +GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) + +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +YTIMG_POOLS = {} of String => YoutubeConnectionPool + # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index b6a2e110..1964d597 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -32,7 +32,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + GGPHT_POOL.client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -80,7 +80,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + get_ytimg_pool(authority).client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -119,7 +119,7 @@ module Invidious::Routes::Images } begin - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + get_ytimg_pool("i9").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex @@ -165,8 +165,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -199,8 +198,7 @@ module Invidious::Routes::Images } begin - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + get_ytimg_pool("i").client &.get(url) do |resp| return request_proc.call(resp) end rescue ex diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index ca612083..26bf2773 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -77,3 +77,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &) client.close end end + +# Fetches a HTTP pool for the specified subdomain of ytimg.com +# +# Creates a new one when the specified pool for the subdomain does not exist +def get_ytimg_pool(subdomain) + if pool = YTIMG_POOLS[subdomain]? + return pool + else + LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"") + pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size) + YTIMG_POOLS[subdomain] = pool + + return pool + end +end From 52bc9aa328e44ff32bb1d7f2e05625e4080459c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:42:40 -0800 Subject: [PATCH 07/70] Refactor duplicate logic in image routes --- src/invidious/routes/images.cr | 97 ++++++++-------------------------- 1 file changed, 21 insertions(+), 76 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 1964d597..7fdd33b0 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -11,29 +11,9 @@ module Invidious::Routes::Images end end - # We're encapsulating this into a proc in order to easily reuse this - # portion of the code for each request block below. - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - env.response.headers.delete("Transfer-Encoding") - return - end - - proxy_file(response, env) - } - begin GGPHT_POOL.client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -61,27 +41,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Connection"] = "close" - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool(authority).client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -101,26 +63,9 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool("i9").client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end @@ -180,28 +125,28 @@ module Invidious::Routes::Images end end - request_proc = ->(response : HTTP::Client::Response) { - env.response.status_code = response.status_code - response.headers.each do |key, value| - if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) - env.response.headers[key] = value - end - end - - env.response.headers["Access-Control-Allow-Origin"] = "*" - - if response.status_code >= 300 && response.status_code != 404 - return env.response.headers.delete("Transfer-Encoding") - end - - proxy_file(response, env) - } - begin get_ytimg_pool("i").client &.get(url) do |resp| - return request_proc.call(resp) + return self.proxy_image(env, resp) end rescue ex end end + + private def self.proxy_image(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) + env.response.headers[key] = value + end + end + + env.response.headers["Access-Control-Allow-Origin"] = "*" + + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") + end + + return proxy_file(response, env) + end end From 06e1a508e8dc5417a61a02ad1eb08e94fb24ae99 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 8 Dec 2023 18:52:11 -0800 Subject: [PATCH 08/70] Fix headers not being added in image requests Regression from #2364 --- src/invidious/routes/images.cr | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 7fdd33b0..c4197746 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -12,7 +12,7 @@ module Invidious::Routes::Images end begin - GGPHT_POOL.client &.get(url) do |resp| + GGPHT_POOL.client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -42,7 +42,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool(authority).client &.get(url) do |resp| + get_ytimg_pool(authority).client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -64,7 +64,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i9").client &.get(url) do |resp| + get_ytimg_pool("i9").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex @@ -110,7 +110,7 @@ module Invidious::Routes::Images if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - if get_ytimg_pool("i9").client &.head(thumbnail_resource_path).status_code == 200 + if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200 name = thumb[:url] + ".jpg" break end @@ -126,7 +126,7 @@ module Invidious::Routes::Images end begin - get_ytimg_pool("i").client &.get(url) do |resp| + get_ytimg_pool("i").client &.get(url, headers) do |resp| return self.proxy_image(env, resp) end rescue ex From 4bc77b81bf994336e324d84ab82a362b330c827d Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 23 Dec 2023 13:47:47 -0800 Subject: [PATCH 09/70] Move YTIMG_POOLS to connection_pool.cr --- src/invidious.cr | 4 --- src/invidious/yt_backend/connection_pool.cr | 32 ++++++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 81db2c6c..e0e72415 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -96,10 +96,6 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size) -# Mapping of subdomain => YoutubeConnectionPool -# This is needed as we may need to access arbitrary subdomains of ytimg -YTIMG_POOLS = {} of String => YoutubeConnectionPool - # CLI Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 26bf2773..646d0d1a 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,17 +1,6 @@ -def add_yt_headers(request) - request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" - request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - - request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" - request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - request.headers["Accept-Language"] ||= "en-us,en;q=0.5" - - # Preserve original cookies and add new YT consent cookie for EU servers - request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" - if !CONFIG.cookies.empty? - request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" - end -end +# Mapping of subdomain => YoutubeConnectionPool +# This is needed as we may need to access arbitrary subdomains of ytimg +private YTIMG_POOLS = {} of String => YoutubeConnectionPool struct YoutubeConnectionPool property! url : URI @@ -54,6 +43,21 @@ struct YoutubeConnectionPool end end +def add_yt_headers(request) + request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" + request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + + request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" + request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + request.headers["Accept-Language"] ||= "en-us,en;q=0.5" + + # Preserve original cookies and add new YT consent cookie for EU servers + request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}" + if !CONFIG.cookies.empty? + request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}" + end +end + def make_client(url : URI, region = nil, force_resolve : Bool = false) client = HTTP::Client.new(url) From 003c6f81dcf6399d1fa808866d0806b915a713ee Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 8 Jan 2024 14:13:38 -0800 Subject: [PATCH 10/70] Preserve connection close header of get_storyboard --- src/invidious/routes/images.cr | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index c4197746..251258ec 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,9 +41,14 @@ module Invidious::Routes::Images end end + # A callable proc to be used inside #proxy_image + callable_proc = ->(env : HTTP::Server::Context) { + env.response.headers["Connection"] = "close" + } + begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp) + return self.proxy_image(env, resp, callable_proc: callable_proc) end rescue ex end @@ -133,7 +138,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response) + private def self.proxy_image(env, response, callable_proc = nil) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -143,6 +148,10 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" + if callable_proc + callable_proc.call(env) + end + if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 75b68618ab14a9f884ee7215a467bc510e8bd2c2 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 25 Apr 2024 13:28:58 -0700 Subject: [PATCH 11/70] Remove useless proc usage in images.cr --- src/invidious/routes/images.cr | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 251258ec..639697db 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -41,14 +41,10 @@ module Invidious::Routes::Images end end - # A callable proc to be used inside #proxy_image - callable_proc = ->(env : HTTP::Server::Context) { - env.response.headers["Connection"] = "close" - } - begin get_ytimg_pool(authority).client &.get(url, headers) do |resp| - return self.proxy_image(env, resp, callable_proc: callable_proc) + env.response.headers["Connection"] = "close" + return self.proxy_image(env, resp) end rescue ex end @@ -138,7 +134,7 @@ module Invidious::Routes::Images end end - private def self.proxy_image(env, response, callable_proc = nil) + private def self.proxy_image(env, response) env.response.status_code = response.status_code response.headers.each do |key, value| if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) @@ -148,10 +144,6 @@ module Invidious::Routes::Images env.response.headers["Access-Control-Allow-Origin"] = "*" - if callable_proc - callable_proc.call(env) - end - if response.status_code >= 300 return env.response.headers.delete("Transfer-Encoding") end From 84e4746265d6077b1537b626c9742498f9cb253c Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 18 Sep 2024 18:14:28 -0300 Subject: [PATCH 12/70] SigHelper: Reconnect to signature helper Signed-off-by: Fijxu --- src/invidious/helpers/sig_helper.cr | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr index 9e72c1c7..6d198a42 100644 --- a/src/invidious/helpers/sig_helper.cr +++ b/src/invidious/helpers/sig_helper.cr @@ -175,8 +175,9 @@ module Invidious::SigHelper @queue = {} of TransactionID => Transaction @conn : Connection + @uri_or_path : String - def initialize(uri_or_path) + def initialize(@uri_or_path) @conn = Connection.new(uri_or_path) listen end @@ -186,10 +187,26 @@ module Invidious::SigHelper LOGGER.debug("SigHelper: Multiplexor listening") - # TODO: reopen socket if unexpectedly closed spawn do loop do - receive_data + begin + receive_data + rescue ex + LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") + # We close the socket because for some reason is not closed. + @conn.close + loop do + begin + @conn = Connection.new(@uri_or_path) + LOGGER.info("SigHelper: Reconnected to SigHelper!") + rescue ex + LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") + sleep 500.milliseconds + next + end + break if !@conn.closed? + end + end Fiber.yield end end From f51a3b8d2b52f83057d0b3be5686149984e66ada Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Wed, 9 Oct 2024 18:07:04 +0200 Subject: [PATCH 13/70] Makefile: Add MT option to enable the 'preview_mt' flag --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 9eb195df..ec22a0de 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,11 @@ STATIC := 0 NO_DBG_SYMBOLS := 0 +# Enable multi-threading. +# Warning: Experimental feature!! +# invidious is not stable when MT is enabled. +MT := 0 + FLAGS ?= @@ -19,6 +24,10 @@ ifeq ($(STATIC), 1) FLAGS += --static endif +ifeq ($(MT), 1) + FLAGS += -Dpreview_mt +endif + ifeq ($(NO_DBG_SYMBOLS), 1) FLAGS += --no-debug From 952b3625a0a8fb21ab04bc267f94a21c331109f6 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 10 Oct 2024 20:31:22 +0200 Subject: [PATCH 14/70] Add "Filipino (auto-generated)" to the list of caption languages --- locales/en-US.json | 1 + src/invidious/videos/caption.cr | 1 + 2 files changed, 2 insertions(+) diff --git a/locales/en-US.json b/locales/en-US.json index 7827d9c6..c23f6bc3 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -286,6 +286,7 @@ "Esperanto": "Esperanto", "Estonian": "Estonian", "Filipino": "Filipino", + "Filipino (auto-generated)": "Filipino (auto-generated)", "Finnish": "Finnish", "French": "French", "French (auto-generated)": "French (auto-generated)", diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index 484e61d2..c811cfe1 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -123,6 +123,7 @@ module Invidious::Videos "Esperanto", "Estonian", "Filipino", + "Filipino (auto-generated)", "Finnish", "French", "French (auto-generated)", From 11ef6703a6e7552681f7ad531e701fe71a79b9e6 Mon Sep 17 00:00:00 2001 From: TheFrenchGhosty <47571719+TheFrenchGhosty@users.noreply.github.com> Date: Sat, 28 Sep 2024 23:54:52 +0200 Subject: [PATCH 15/70] Remove myself from CODEOWNERS on the config file (#4942) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7a2c3760..9ca09368 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,7 +6,7 @@ docker/ @unixfox kubernetes/ @unixfox README.md @thefrenchghosty -config/config.example.yml @thefrenchghosty @SamantazFox @unixfox +config/config.example.yml @SamantazFox @unixfox scripts/ @syeopite shards.lock @syeopite From eba06bce3521d734a90397d2b987fa04c8729c71 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 24 Sep 2023 15:09:59 -0400 Subject: [PATCH 16/70] Refactor instance fetching logic into separate job --- src/invidious.cr | 2 + src/invidious/helpers/utils.cr | 62 -------------- src/invidious/jobs/instance_refresh_job.cr | 94 ++++++++++++++++++++++ src/invidious/routes/misc.cr | 11 ++- 4 files changed, 106 insertions(+), 63 deletions(-) create mode 100644 src/invidious/jobs/instance_refresh_job.cr diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..63f2a9cc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -189,6 +189,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + Invidious::Jobs.start_all def popular_videos diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8e9e9a6a..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -323,68 +323,6 @@ def parse_range(range) return 0_i64, nil end -def fetch_random_instance - begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) - - # Timeouts - instance_api_client.connect_timeout = 10.seconds - instance_api_client.dns_timeout = 10.seconds - - instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a - instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException - instance_list = [] of JSON::Any - end - - filtered_instance_list = [] of String - - instance_list.each do |data| - # TODO Check if current URL is onion instance and use .onion types if so. - if data[1]["type"] == "https" - # Instances can have statistics disabled, which is an requirement of version validation. - # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails. - begin - data[1]["stats"].as_nil - next - rescue TypeCastError - end - - # stats endpoint could also lack the software dict. - next if data[1]["stats"]["software"]?.nil? - - # Makes sure the instance isn't too outdated. - if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] - remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) - next if !remote_commit_date - - remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) - local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) - - next if (remote_commit_date - local_commit_date).abs.days > 30 - - begin - data[1]["monitor"].as_nil - health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] - filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 - rescue TypeCastError - # We can't check the health if the monitoring is broken. Thus we'll just add it to the list - # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that - # it's an error that often occurs with all the instances at the same time, we have to just skip the check. - filtered_instance_list << data[0].as_s - end - end - end - end - - # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io - if filtered_instance_list.size == 0 - return "redirect.invidious.io" - end - - return filtered_instance_list.sample(1)[0] -end - def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String str = uri.to_s.sub(/^https?:\/\//, "") if str.size > max_length diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr new file mode 100644 index 00000000..bfda9f3f --- /dev/null +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -0,0 +1,94 @@ +class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob + # We update the internals of a constant as so it can be accessed from anywhere + # within the codebase + # + # "INSTANCES" => Array(Tuple(String, String)) # region, instance + + INSTANCES = {"INSTANCES" => [] of Tuple(String, String)} + + def initialize + end + + def begin + loop do + refresh_instances + LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes") + sleep 30.minute + Fiber.yield + end + end + + # Refreshes the list of instances used for redirects. + # + # Does the following three checks for each instance + # - Is it a clear-net instance? + # - Is it an instance with a good uptime? + # - Is it an updated instance? + private def refresh_instances + raw_instance_list = self.fetch_instances + filtered_instance_list = [] of Tuple(String, String) + + raw_instance_list.each do |instance_data| + # TODO allow Tor hidden service instances when the current instance + # is also a hidden service. Same for i2p and any other non-clearnet instances. + begin + domain = instance_data[0] + info = instance_data[1] + stats = info["stats"] + + next unless info["type"] == "https" + next if bad_uptime?(info["monitor"]) + next if outdated?(stats["software"]["version"]) + + filtered_instance_list << {info["region"].as_s, domain.as_s} + rescue ex + if domain + LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + else + LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + end + end + end + + if !filtered_instance_list.empty? + INSTANCES["INSTANCES"] = filtered_instance_list + end + end + + # Fetches information regarding instances from api.invidious.io or an otherwise configured URL + private def fetch_instances : Array(JSON::Any) + begin + instance_api_client = make_client(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException + raw_instance_list = [] of JSON::Any + end + + return raw_instance_list + end + + # Checks if the given target instance is outdated + private def outdated?(target_instance_version) : Bool + remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + return false if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + return (remote_commit_date - local_commit_date).abs.days > 30 + end + + # Checks if the uptime of the target instance is greater than 90% over a 30 day period + private def bad_uptime?(target_instance_health_monitor) : Bool + return false if !target_instance_health_monitor["statusClass"] == "success" + return false if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + + return true + end +end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index d6bd9571..8b620d63 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -40,7 +40,16 @@ module Invidious::Routes::Misc def self.cross_instance_redirect(env) referer = get_referer(env) - instance_url = fetch_random_instance + + instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"] + if instance_list.empty? + instance_url = "redirect.invidious.io" + else + # Sample returns an array + # Instances are packaged as {region, domain} in the instance list + instance_url = instance_list.sample(1)[0][1] + end + env.redirect "https://#{instance_url}#{referer}" end end From a5d11daf85192abbd7e5a6f7c99b1bb09343e268 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Dec 2023 11:21:06 -0800 Subject: [PATCH 17/70] Use HTTP::Client directly in instance list job The HTTP::Client created via `make_client` is affected by the force_resolve configuration option. However, api.invidious.io does not support ipv6 and as such any request with ipv6 to api.invidious.io will instead raise. Directly calling the HTTP::Client will ignore the force_resolve option allowing requests to go through ipv4 when needed. --- src/invidious/jobs/instance_refresh_job.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index bfda9f3f..38071998 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -58,7 +58,10 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Fetches information regarding instances from api.invidious.io or an otherwise configured URL private def fetch_instances : Array(JSON::Any) begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) + # We directly call the stdlib HTTP::Client here as it allows us to negate the effects + # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6 + # and as such the following request raises if we were to use force_resolve with the ipv6 value. + instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) # Timeouts instance_api_client.connect_timeout = 10.seconds From 2a783b847a9d069d2fda6bf0d260494167f2a9b3 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Dec 2023 11:43:44 -0800 Subject: [PATCH 18/70] Fix invalid logic for instance uptime comparison --- src/invidious/jobs/instance_refresh_job.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index 38071998..b385d45c 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -69,7 +69,7 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException + rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException raw_instance_list = [] of JSON::Any end @@ -89,9 +89,9 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Checks if the uptime of the target instance is greater than 90% over a 30 day period private def bad_uptime?(target_instance_health_monitor) : Bool - return false if !target_instance_health_monitor["statusClass"] == "success" - return false if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + return true if !target_instance_health_monitor["statusClass"] == "success" + return true if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 - return true + return false end end From 06629d375f80d8ef0dcf2b1af772defddc5f05da Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:28:15 -0700 Subject: [PATCH 19/70] Update uptime logic to handle updown.io response --- src/invidious/jobs/instance_refresh_job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index b385d45c..cb4280b9 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -89,8 +89,8 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Checks if the uptime of the target instance is greater than 90% over a 30 day period private def bad_uptime?(target_instance_health_monitor) : Bool - return true if !target_instance_health_monitor["statusClass"] == "success" - return true if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + return true if !target_instance_health_monitor["down"].as_bool == false + return true if target_instance_health_monitor["uptime"].as_f < 90 return false end From 5b1232a3145acf89350942460aa79539e7957881 Mon Sep 17 00:00:00 2001 From: Thomas Lange Date: Tue, 26 Sep 2023 21:45:52 +0200 Subject: [PATCH 20/70] Add option to control preloading of video data Fix #4110 by adding an option to control the preloading of video data on page load. If disabled ("false"), the browser will not preload any video data until the user explicitly hits the "Play" button. If enabled ("true"), the default behavior will be used, which means the browser decides how much of the video will be preloaded. --- assets/js/player.js | 5 ++++- config/config.example.yml | 14 ++++++++++++++ locales/de.json | 1 + locales/en-US.json | 1 + src/invidious/config.cr | 1 + src/invidious/routes/preferences.cr | 5 +++++ src/invidious/user/preferences.cr | 1 + src/invidious/videos/video_preferences.cr | 6 ++++++ src/invidious/views/components/player.ecr | 4 +++- src/invidious/views/user/preferences.ecr | 5 +++++ 10 files changed, 41 insertions(+), 2 deletions(-) diff --git a/assets/js/player.js b/assets/js/player.js index d32062c6..9b8cf1af 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -3,7 +3,6 @@ var player_data = JSON.parse(document.getElementById('player_data').textContent) var video_data = JSON.parse(document.getElementById('video_data').textContent); var options = { - preload: 'auto', liveui: true, playbackRates: [0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], controlBar: { @@ -35,6 +34,10 @@ if (player_data.aspect_ratio) { options.aspectRatio = player_data.aspect_ratio; } +if (player_data.preload) { + options.preload = player_data.preload +} + var embed_url = new URL(location); embed_url.searchParams.delete('v'); var short_url = location.origin + '/' + video_data.id + embed_url.search; diff --git a/config/config.example.yml b/config/config.example.yml index 219aa03f..e2aaec6c 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -707,6 +707,20 @@ default_user_preferences: # Video player behavior # ----------------------------- + ## + ## Automatically preload video on page load. This option controls the + ## value for the "preload" attribute of the HTML5