From 11ef6703a6e7552681f7ad531e701fe71a79b9e6 Mon Sep 17 00:00:00 2001 From: TheFrenchGhosty <47571719+TheFrenchGhosty@users.noreply.github.com> Date: Sat, 28 Sep 2024 23:54:52 +0200 Subject: [PATCH 01/49] Remove myself from CODEOWNERS on the config file (#4942) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7a2c3760..9ca09368 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -6,7 +6,7 @@ docker/ @unixfox kubernetes/ @unixfox README.md @thefrenchghosty -config/config.example.yml @thefrenchghosty @SamantazFox @unixfox +config/config.example.yml @SamantazFox @unixfox scripts/ @syeopite shards.lock @syeopite From eba06bce3521d734a90397d2b987fa04c8729c71 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 24 Sep 2023 15:09:59 -0400 Subject: [PATCH 02/49] Refactor instance fetching logic into separate job --- src/invidious.cr | 2 + src/invidious/helpers/utils.cr | 62 -------------- src/invidious/jobs/instance_refresh_job.cr | 94 ++++++++++++++++++++++ src/invidious/routes/misc.cr | 11 ++- 4 files changed, 106 insertions(+), 63 deletions(-) create mode 100644 src/invidious/jobs/instance_refresh_job.cr diff --git a/src/invidious.cr b/src/invidious.cr index 3804197e..63f2a9cc 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -189,6 +189,8 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new + Invidious::Jobs.start_all def popular_videos diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 8e9e9a6a..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -323,68 +323,6 @@ def parse_range(range) return 0_i64, nil end -def fetch_random_instance - begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) - - # Timeouts - instance_api_client.connect_timeout = 10.seconds - instance_api_client.dns_timeout = 10.seconds - - instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a - instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException - instance_list = [] of JSON::Any - end - - filtered_instance_list = [] of String - - instance_list.each do |data| - # TODO Check if current URL is onion instance and use .onion types if so. - if data[1]["type"] == "https" - # Instances can have statistics disabled, which is an requirement of version validation. - # as_nil? doesn't exist. Thus we'll have to handle the error raised if as_nil fails. - begin - data[1]["stats"].as_nil - next - rescue TypeCastError - end - - # stats endpoint could also lack the software dict. - next if data[1]["stats"]["software"]?.nil? - - # Makes sure the instance isn't too outdated. - if remote_version = data[1]["stats"]?.try &.["software"]?.try &.["version"] - remote_commit_date = remote_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) - next if !remote_commit_date - - remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) - local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) - - next if (remote_commit_date - local_commit_date).abs.days > 30 - - begin - data[1]["monitor"].as_nil - health = data[1]["monitor"].as_h["dailyRatios"][0].as_h["ratio"] - filtered_instance_list << data[0].as_s if health.to_s.to_f > 90 - rescue TypeCastError - # We can't check the health if the monitoring is broken. Thus we'll just add it to the list - # and move on. Ideally we'll ignore any instance that has broken health monitoring but due to the fact that - # it's an error that often occurs with all the instances at the same time, we have to just skip the check. - filtered_instance_list << data[0].as_s - end - end - end - end - - # If for some reason no instances managed to get fetched successfully then we'll just redirect to redirect.invidious.io - if filtered_instance_list.size == 0 - return "redirect.invidious.io" - end - - return filtered_instance_list.sample(1)[0] -end - def reduce_uri(uri : URI | String, max_length : Int32 = 50, suffix : String = "…") : String str = uri.to_s.sub(/^https?:\/\//, "") if str.size > max_length diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr new file mode 100644 index 00000000..bfda9f3f --- /dev/null +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -0,0 +1,94 @@ +class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob + # We update the internals of a constant as so it can be accessed from anywhere + # within the codebase + # + # "INSTANCES" => Array(Tuple(String, String)) # region, instance + + INSTANCES = {"INSTANCES" => [] of Tuple(String, String)} + + def initialize + end + + def begin + loop do + refresh_instances + LOGGER.info("InstanceListRefreshJob: Done, sleeping for 30 minutes") + sleep 30.minute + Fiber.yield + end + end + + # Refreshes the list of instances used for redirects. + # + # Does the following three checks for each instance + # - Is it a clear-net instance? + # - Is it an instance with a good uptime? + # - Is it an updated instance? + private def refresh_instances + raw_instance_list = self.fetch_instances + filtered_instance_list = [] of Tuple(String, String) + + raw_instance_list.each do |instance_data| + # TODO allow Tor hidden service instances when the current instance + # is also a hidden service. Same for i2p and any other non-clearnet instances. + begin + domain = instance_data[0] + info = instance_data[1] + stats = info["stats"] + + next unless info["type"] == "https" + next if bad_uptime?(info["monitor"]) + next if outdated?(stats["software"]["version"]) + + filtered_instance_list << {info["region"].as_s, domain.as_s} + rescue ex + if domain + LOGGER.info("InstanceListRefreshJob: failed to parse information from '#{domain}' because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + else + LOGGER.info("InstanceListRefreshJob: failed to parse information from an instance because \"#{ex}\"\n\"#{ex.backtrace.join('\n')}\" ") + end + end + end + + if !filtered_instance_list.empty? + INSTANCES["INSTANCES"] = filtered_instance_list + end + end + + # Fetches information regarding instances from api.invidious.io or an otherwise configured URL + private def fetch_instances : Array(JSON::Any) + begin + instance_api_client = make_client(URI.parse("https://api.invidious.io")) + + # Timeouts + instance_api_client.connect_timeout = 10.seconds + instance_api_client.dns_timeout = 10.seconds + + raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a + instance_api_client.close + rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException + raw_instance_list = [] of JSON::Any + end + + return raw_instance_list + end + + # Checks if the given target instance is outdated + private def outdated?(target_instance_version) : Bool + remote_commit_date = target_instance_version.as_s.match(/\d{4}\.\d{2}\.\d{2}/) + return false if !remote_commit_date + + remote_commit_date = Time.parse(remote_commit_date[0], "%Y.%m.%d", Time::Location::UTC) + local_commit_date = Time.parse(CURRENT_VERSION, "%Y.%m.%d", Time::Location::UTC) + + return (remote_commit_date - local_commit_date).abs.days > 30 + end + + # Checks if the uptime of the target instance is greater than 90% over a 30 day period + private def bad_uptime?(target_instance_health_monitor) : Bool + return false if !target_instance_health_monitor["statusClass"] == "success" + return false if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + + return true + end +end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index d6bd9571..8b620d63 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -40,7 +40,16 @@ module Invidious::Routes::Misc def self.cross_instance_redirect(env) referer = get_referer(env) - instance_url = fetch_random_instance + + instance_list = Invidious::Jobs::InstanceListRefreshJob::INSTANCES["INSTANCES"] + if instance_list.empty? + instance_url = "redirect.invidious.io" + else + # Sample returns an array + # Instances are packaged as {region, domain} in the instance list + instance_url = instance_list.sample(1)[0][1] + end + env.redirect "https://#{instance_url}#{referer}" end end From a5d11daf85192abbd7e5a6f7c99b1bb09343e268 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Dec 2023 11:21:06 -0800 Subject: [PATCH 03/49] Use HTTP::Client directly in instance list job The HTTP::Client created via `make_client` is affected by the force_resolve configuration option. However, api.invidious.io does not support ipv6 and as such any request with ipv6 to api.invidious.io will instead raise. Directly calling the HTTP::Client will ignore the force_resolve option allowing requests to go through ipv4 when needed. --- src/invidious/jobs/instance_refresh_job.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index bfda9f3f..38071998 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -58,7 +58,10 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Fetches information regarding instances from api.invidious.io or an otherwise configured URL private def fetch_instances : Array(JSON::Any) begin - instance_api_client = make_client(URI.parse("https://api.invidious.io")) + # We directly call the stdlib HTTP::Client here as it allows us to negate the effects + # of the force_resolve config option. This is needed as api.invidious.io does not support ipv6 + # and as such the following request raises if we were to use force_resolve with the ipv6 value. + instance_api_client = HTTP::Client.new(URI.parse("https://api.invidious.io")) # Timeouts instance_api_client.connect_timeout = 10.seconds From 2a783b847a9d069d2fda6bf0d260494167f2a9b3 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 7 Dec 2023 11:43:44 -0800 Subject: [PATCH 04/49] Fix invalid logic for instance uptime comparison --- src/invidious/jobs/instance_refresh_job.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index 38071998..b385d45c 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -69,7 +69,7 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob raw_instance_list = JSON.parse(instance_api_client.get("/instances.json").body).as_a instance_api_client.close - rescue Socket::ConnectError | IO::TimeoutError | JSON::ParseException + rescue ex : Socket::ConnectError | IO::TimeoutError | JSON::ParseException raw_instance_list = [] of JSON::Any end @@ -89,9 +89,9 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Checks if the uptime of the target instance is greater than 90% over a 30 day period private def bad_uptime?(target_instance_health_monitor) : Bool - return false if !target_instance_health_monitor["statusClass"] == "success" - return false if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + return true if !target_instance_health_monitor["statusClass"] == "success" + return true if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 - return true + return false end end From 06629d375f80d8ef0dcf2b1af772defddc5f05da Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:28:15 -0700 Subject: [PATCH 05/49] Update uptime logic to handle updown.io response --- src/invidious/jobs/instance_refresh_job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/jobs/instance_refresh_job.cr b/src/invidious/jobs/instance_refresh_job.cr index b385d45c..cb4280b9 100644 --- a/src/invidious/jobs/instance_refresh_job.cr +++ b/src/invidious/jobs/instance_refresh_job.cr @@ -89,8 +89,8 @@ class Invidious::Jobs::InstanceListRefreshJob < Invidious::Jobs::BaseJob # Checks if the uptime of the target instance is greater than 90% over a 30 day period private def bad_uptime?(target_instance_health_monitor) : Bool - return true if !target_instance_health_monitor["statusClass"] == "success" - return true if target_instance_health_monitor["30dRatio"]["ratio"].as_s.to_f < 90 + return true if !target_instance_health_monitor["down"].as_bool == false + return true if target_instance_health_monitor["uptime"].as_f < 90 return false end From 5b1232a3145acf89350942460aa79539e7957881 Mon Sep 17 00:00:00 2001 From: Thomas Lange Date: Tue, 26 Sep 2023 21:45:52 +0200 Subject: [PATCH 06/49] Add option to control preloading of video data Fix #4110 by adding an option to control the preloading of video data on page load. If disabled ("false"), the browser will not preload any video data until the user explicitly hits the "Play" button. If enabled ("true"), the default behavior will be used, which means the browser decides how much of the video will be preloaded. --- assets/js/player.js | 5 ++++- config/config.example.yml | 14 ++++++++++++++ locales/de.json | 1 + locales/en-US.json | 1 + src/invidious/config.cr | 1 + src/invidious/routes/preferences.cr | 5 +++++ src/invidious/user/preferences.cr | 1 + src/invidious/videos/video_preferences.cr | 6 ++++++ src/invidious/views/components/player.ecr | 4 +++- src/invidious/views/user/preferences.ecr | 5 +++++ 10 files changed, 41 insertions(+), 2 deletions(-) diff --git a/assets/js/player.js b/assets/js/player.js index d32062c6..9b8cf1af 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -3,7 +3,6 @@ var player_data = JSON.parse(document.getElementById('player_data').textContent) var video_data = JSON.parse(document.getElementById('video_data').textContent); var options = { - preload: 'auto', liveui: true, playbackRates: [0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], controlBar: { @@ -35,6 +34,10 @@ if (player_data.aspect_ratio) { options.aspectRatio = player_data.aspect_ratio; } +if (player_data.preload) { + options.preload = player_data.preload +} + var embed_url = new URL(location); embed_url.searchParams.delete('v'); var short_url = location.origin + '/' + video_data.id + embed_url.search; diff --git a/config/config.example.yml b/config/config.example.yml index 219aa03f..e2aaec6c 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -707,6 +707,20 @@ default_user_preferences: # Video player behavior # ----------------------------- + ## + ## Automatically preload video on page load. This option controls the + ## value for the "preload" attribute of the HTML5