diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9f17bb40..9ca09368 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,6 @@ +# Default and lowest precedence. If none of the below matches, @iv-org/developers would be requested for review. +* @iv-org/developers + docker-compose.yml @unixfox docker/ @unixfox kubernetes/ @unixfox diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 02bc3795..4c1a6330 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -10,10 +10,8 @@ assignees: '' -
#{issue_template}+
#{issue_template}END_HTML @@ -139,7 +130,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, exception : Exception, - additional_fields : Hash(String, Object) | Nil = nil, + additional_fields : Hash(String, Object) | Nil = nil ) if exception.is_a?(InfoException) return error_json_helper(env, status_code, exception.message || "", additional_fields) @@ -161,7 +152,7 @@ def error_json_helper( env : HTTP::Server::Context, status_code : Int32, message : String, - additional_fields : Hash(String, Object) | Nil = nil, + additional_fields : Hash(String, Object) | Nil = nil ) env.response.content_type = "application/json" env.response.status_code = status_code diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 13ea9fe9..f3e3b951 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr @@ -27,7 +27,6 @@ class Kemal::RouteHandler # Processes the route if it's a match. Otherwise renders 404. private def process_request(context) raise Kemal::Exceptions::RouteNotFound.new(context) unless context.route_found? - return if context.response.closed? content = context.route.handler.call(context) if !Kemal.config.error_handlers.empty? && Kemal.config.error_handlers.has_key?(context.response.status_code) && exclude_match?(context) diff --git a/src/invidious/helpers/i18n.cr b/src/invidious/helpers/i18n.cr index bca2edda..1ba3ea61 100644 --- a/src/invidious/helpers/i18n.cr +++ b/src/invidious/helpers/i18n.cr @@ -54,7 +54,6 @@ LOCALES_LIST = { "sr" => "Srpski (latinica)", # Serbian (Latin) "sr_Cyrl" => "Српски (ћирилица)", # Serbian (Cyrillic) "sv-SE" => "Svenska", # Swedish - "ta" => "தமிழ்", # Tamil "tr" => "Türkçe", # Turkish "uk" => "Українська", # Ukrainian "vi" => "Tiếng Việt", # Vietnamese diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index 84847321..43e7171b 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -55,11 +55,12 @@ macro templated(_filename, template = "template", navbar_search = true) {{ layout = "src/invidious/views/" + template + ".ecr" }} __content_filename__ = {{filename}} - render {{filename}}, {{layout}} + content = Kilt.render({{filename}}) + Kilt.render({{layout}}) end macro rendered(filename) - render("src/invidious/views/#{{{filename}}}.ecr") + Kilt.render("src/invidious/views/#{{{filename}}}.ecr") end # Similar to Kemals halt method but works in a diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 2796a8dc..1fef5f93 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -24,7 +24,6 @@ struct SearchVideo property length_seconds : Int32 property premiere_timestamp : Time? property author_verified : Bool - property author_thumbnail : String? property badges : VideoBadges def to_xml(auto_generated, query_params, xml : XML::Builder) @@ -89,24 +88,6 @@ struct SearchVideo json.field "authorUrl", "/channel/#{self.ucid}" json.field "authorVerified", self.author_verified - author_thumbnail = self.author_thumbnail - - if author_thumbnail - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} - - qualities.each do |quality| - json.object do - json.field "url", author_thumbnail.gsub(/=s\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end - end - end - end - end - json.field "videoThumbnails" do Invidious::JSONify::APIv1.thumbnails(json, self.id) end @@ -242,7 +223,7 @@ struct SearchChannel qualities.each do |quality| json.object do - json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}") + json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}") json.field "width", quality json.field "height", quality end @@ -291,55 +272,6 @@ struct SearchHashtag end end -# A `ProblematicTimelineItem` is a `SearchItem` created by Invidious that -# represents an item that caused an exception during parsing. -# -# This is not a parsed object from YouTube but rather an Invidious-only type -# created to gracefully communicate parse errors without throwing away -# the rest of the (hopefully) successfully parsed item on a page. -struct ProblematicTimelineItem - property parse_exception : Exception - property id : String - - def initialize(@parse_exception) - @id = Random.new.hex(8) - end - - def to_json(locale : String?, json : JSON::Builder) - json.object do - json.field "type", "parse-error" - json.field "errorMessage", @parse_exception.message - json.field "errorBacktrace", @parse_exception.inspect_with_backtrace - end - end - - # Provides compatibility with PlaylistVideo - def to_json(json : JSON::Builder, *args, **kwargs) - return to_json("", json) - end - - def to_xml(env, locale, xml : XML::Builder) - xml.element("entry") do - xml.element("id") { xml.text "iv-err-#{@id}" } - xml.element("title") { xml.text "Parse Error: This item has failed to parse" } - xml.element("updated") { xml.text Time.utc.to_rfc3339 } - - xml.element("content", type: "xhtml") do - xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do - xml.element("div") do - xml.element("h4") { translate(locale, "timeline_parse_error_placeholder_heading") } - xml.element("p") { translate(locale, "timeline_parse_error_placeholder_message") } - end - - xml.element("pre") do - get_issue_template(env, @parse_exception) - end - end - end - end - end -end - class Category include DB::Serializable @@ -382,4 +314,4 @@ struct Continuation end end -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category | ProblematicTimelineItem +alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 5637e533..4d9bb28d 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -262,7 +262,7 @@ def get_referer(env, fallback = "/", unroll = true) end referer = referer.request_target - referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z+]/, "").lstrip("/\\") + referer = "/" + referer.gsub(/[^\/?@&%=\-_.:,*0-9a-zA-Z]/, "").lstrip("/\\") if referer == env.request.path referer = fallback @@ -383,22 +383,3 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String) end return text end - -def encrypt_ecb_without_salt(data, key) - cipher = OpenSSL::Cipher.new("aes-128-ecb") - cipher.encrypt - cipher.key = key - - io = IO::Memory.new - io.write(cipher.update(data)) - io.write(cipher.final) - io.rewind - - return io -end - -def invidious_companion_encrypt(data) - timestamp = Time.utc.to_unix - encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key) - return Base64.urlsafe_encode(encrypted_data) -end diff --git a/src/invidious/jobs/notification_job.cr b/src/invidious/jobs/notification_job.cr index 968ee47f..b445107b 100644 --- a/src/invidious/jobs/notification_job.cr +++ b/src/invidious/jobs/notification_job.cr @@ -1,32 +1,8 @@ -struct VideoNotification - getter video_id : String - getter channel_id : String - getter published : Time - - def_hash @channel_id, @video_id - - def ==(other) - video_id == other.video_id - end - - def self.from_video(video : ChannelVideo) : self - VideoNotification.new(video.id, video.ucid, video.published) - end - - def initialize(@video_id, @channel_id, @published) - end - - def clone : VideoNotification - VideoNotification.new(video_id.clone, channel_id.clone, published.clone) - end -end - class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob - private getter notification_channel : ::Channel(VideoNotification) private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)}) private getter pg_url : URI - def initialize(@notification_channel, @connection_channel, @pg_url) + def initialize(@connection_channel, @pg_url) end def begin @@ -34,70 +10,6 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) } - # hash of channels to their videos (id+published) that need notifying - to_notify = Hash(String, Set(VideoNotification)).new( - ->(hash : Hash(String, Set(VideoNotification)), key : String) { - hash[key] = Set(VideoNotification).new - } - ) - notify_mutex = Mutex.new - - # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job) - spawn do - begin - loop do - notification = notification_channel.receive - notify_mutex.synchronize do - to_notify[notification.channel_id] << notification - end - end - end - end - # fiber to regularly persist all cached notifications - spawn do - loop do - begin - LOGGER.debug("NotificationJob: waking up") - cloned = {} of String => Set(VideoNotification) - notify_mutex.synchronize do - cloned = to_notify.clone - to_notify.clear - end - - cloned.each do |channel_id, notifications| - if notifications.empty? - next - end - - LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications") - if CONFIG.enable_user_notifications - video_ids = notifications.map(&.video_id) - Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids) - PG_DB.using_connection do |conn| - notifications.each do |n| - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => n.channel_id, - "videoId" => n.video_id, - "published" => n.published.to_unix, - }.to_json - conn.exec("NOTIFY notifications, E'#{payload}'") - end - end - else - Invidious::Database::Users.feed_needs_update(channel_id) - end - end - - LOGGER.trace("NotificationJob: Done, sleeping") - rescue ex - LOGGER.error("NotificationJob: #{ex.message}") - end - sleep 1.minute - Fiber.yield - end - end - loop do action, connection = connection_channel.receive diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 58805af2..08cd533f 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -267,12 +267,6 @@ module Invidious::JSONify::APIv1 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 - json.field "published", rv["published"]? - if rv["published"]?.try &.presence - json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale)) - else - json.field "publishedText", "" - end end end end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 28ff0ff6..823ca85b 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -81,7 +81,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) }) end -def template_mix(mix, listen) +def template_mix(mix) html = <<-END_HTML
#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7c584d15..a51e88b4 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -432,7 +432,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -448,7 +448,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo | ProblematicTimelineItem + videos = [] of PlaylistVideo if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -500,14 +500,12 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) index: index, }) end - rescue ex - videos << ProblematicTimelineItem.new(parse_exception: ex) end return videos end -def template_playlist(playlist, listen) +def template_playlist(playlist) html = <<-END_HTML#{recode_length_seconds(video["lengthSeconds"].as_i)}
diff --git a/src/invidious/routes/account.cr b/src/invidious/routes/account.cr index c8db207c..dd65e7a6 100644 --- a/src/invidious/routes/account.cr +++ b/src/invidious/routes/account.cr @@ -328,9 +328,17 @@ module Invidious::Routes::Account end end - case action = env.params.query["action"]? - when "revoke_token" - session = env.params.query["session"] + if env.params.query["action_revoke_token"]? + action = "action_revoke_token" + else + return env.redirect referer + end + + session = env.params.query["session"]? + session ||= "" + + case action + when .starts_with? "action_revoke_token" Invidious::Database::SessionIDs.delete(sid: session, email: user.email) else return error_json(400, "Unsupported action #{action}") diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index c27caad7..d89e752c 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -8,11 +8,6 @@ module Invidious::Routes::API::Manifest id = env.params.url["id"] region = env.params.query["region"]? - if CONFIG.invidious_companion.present? - invidious_companion = CONFIG.invidious_companion.sample - return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}" - end - # Since some implementations create playlists based on resolution regardless of different codecs, # we can opt to only add a source to a representation if it has a unique height within that representation unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } @@ -32,21 +27,28 @@ module Invidious::Routes::API::Manifest haltf env, status_code: response.status_code end - # Proxy URLs for video playback on invidious. - # Other API clients can get the original URLs by omiting `local=true`. - manifest = response.body.gsub(/<%=translate(locale, "timeline_parse_error_placeholder_message")%>
-<%=get_issue_template(env, item.parse_exception)[1]%>-