Add conversation-based forwarding for limited visibility statuses through bearcaps

16 files changed, 267 insertions, 58 deletions

diff --git a/app/controllers/activitypub/contexts_controller.rb b/app/controllers/activitypub/contexts_controller.rb
new file mode 100644
index 000000000..0d3034989
--- /dev/null
+++ b/app/controllers/activitypub/contexts_controller.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class ActivityPub::ContextsController < ActivityPub::BaseController
+  before_action :set_conversation
+
+  def show
+    expires_in 3.minutes, public: public_fetch_mode?
+    render_with_cache json: @conversation, serializer: ActivityPub::ContextSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
+  end
+
+  private
+
+  def set_conversation
+    @conversation = Conversation.local.find(params[:id])
+  end
+end
diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb
index 189b92012..ec117f144 100644
--- a/app/controllers/concerns/cache_concern.rb
+++ b/app/controllers/concerns/cache_concern.rb
@@ -25,7 +25,7 @@ module CacheConcern
   end
 
   def set_cache_headers
-    response.headers['Vary'] = public_fetch_mode? ? 'Accept' : 'Accept, Signature'
+    response.headers['Vary'] = public_fetch_mode? ? 'Accept, Authorization' : 'Accept, Signature, Authorization'
   end
 
   def cache_collection(raw, klass)
diff --git a/app/controllers/statuses_controller.rb b/app/controllers/statuses_controller.rb
index 17ddd31fb..63bd82bd7 100644
--- a/app/controllers/statuses_controller.rb
+++ b/app/controllers/statuses_controller.rb
@@ -66,7 +66,12 @@ class StatusesController < ApplicationController
 
   def set_status
     @status = @account.statuses.find(params[:id])
-    authorize @status, :show?
+
+    if request.authorization.present? && request.authorization.match(/^Bearer /i)
+      raise Mastodon::NotPermittedError unless @status.capability_tokens.find_by(token: request.authorization.gsub(/^Bearer /i, ''))
+    else
+      authorize @status, :show?
+    end
   rescue Mastodon::NotPermittedError
     not_found
   end
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index 1c473efa3..8bda1548c 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -49,13 +49,12 @@ module JsonLdHelper
     !uri.start_with?('http://', 'https://')
   end
 
-  def invalid_origin?(url)
-    return true if unsupported_uri_scheme?(url)
-
-    needle   = Addressable::URI.parse(url).host
-    haystack = Addressable::URI.parse(@account.uri).host
+  def same_origin?(url_a, url_b)
+    Addressable::URI.parse(url_a).host.casecmp(Addressable::URI.parse(url_b).host).zero?
+  end
 
-    !haystack.casecmp(needle).zero?
+  def invalid_origin?(url)
+    unsupported_uri_scheme?(url) || !same_origin?(url, @account.uri)
   end
 
   def canonicalize(json)
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index f275feefc..1ab239757 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -90,6 +90,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     fetch_replies(@status)
     check_for_spam
     distribute(@status)
+    forward_for_conversation
     forward_for_reply
   end
 
@@ -114,7 +115,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
         sensitive: @object['sensitive'] || false,
         visibility: visibility_from_audience,
         thread: replied_to_status,
-        conversation: conversation_from_uri(@object['conversation']),
+        conversation: conversation_from_context,
         media_attachment_ids: process_attachments.take(4).map(&:id),
         poll: process_poll,
       }
@@ -122,8 +123,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   end
 
   def process_audience
+    conversation_uri = value_or_id(@object['context'])
+
     (audience_to + audience_cc).uniq.each do |audience|
-      next if audience == ActivityPub::TagManager::COLLECTIONS[:public]
+      next if audience == ActivityPub::TagManager::COLLECTIONS[:public] || audience == conversation_uri
 
       # Unlike with tags, there is no point in resolving accounts we don't already
       # know here, because silent mentions would only be used for local access
@@ -340,15 +343,45 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
   end
 
-  def conversation_from_uri(uri)
-    return nil if uri.nil?
-    return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
+  def conversation_from_context
+    atom_uri = @object['conversation']
 
-    begin
-      Conversation.find_or_create_by!(uri: uri)
-    rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
-      retry
+    conversation = begin
+      if atom_uri.present? && OStatus::TagManager.instance.local_id?(atom_uri)
+        Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(atom_uri, 'Conversation'))
+      elsif atom_uri.present? && @object['context'].present?
+        Conversation.find_by(uri: atom_uri)
+      elsif atom_uri.present?
+        begin
+          Conversation.find_or_create_by!(uri: atom_uri)
+        rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
+          retry
+        end
+      end
     end
+
+    return conversation if @object['context'].nil?
+
+    uri            = value_or_id(@object['context'])
+    conversation ||= ActivityPub::TagManager.instance.uri_to_resource(uri, Conversation)
+
+    return conversation if (conversation.present? && conversation.uri == uri) || !uri.start_with?('https://')
+
+    conversation_json = begin
+      if @object['context'].is_a?(Hash) && !invalid_origin?(uri)
+        @object['context']
+      else
+        fetch_resource(uri, true)
+      end
+    end
+
+    return conversation if conversation_json.blank?
+
+    conversation ||= Conversation.new
+    conversation.uri = uri
+    conversation.inbox_url = conversation_json['inbox']
+    conversation.save! if conversation.changed?
+    conversation
   end
 
   def visibility_from_audience
@@ -492,6 +525,12 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     SpamCheck.perform(@status)
   end
 
+  def forward_for_conversation
+    return unless audience_to.include?(value_or_id(@object['context'])) && @json['signature'].present? && @status.conversation.local?
+
+    ActivityPub::ForwardDistributionWorker.perform_async(@status.conversation_id, Oj.dump(@json))
+  end
+
   def forward_for_reply
     return unless @status.distributable? && @json['signature'].present? && reply_to_local?
 
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 3f98dad2e..b29c606e9 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -21,8 +21,11 @@ class ActivityPub::TagManager
     when :person
       target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
-      return activity_account_status_url(target.account, target) if target.reblog?
-      short_account_status_url(target.account, target)
+      if target.reblog?
+        activity_account_status_url(target.account, target)
+      else
+        short_account_status_url(target.account, target)
+      end
     end
   end
 
@@ -33,10 +36,15 @@ class ActivityPub::TagManager
     when :person
       target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
-      return activity_account_status_url(target.account, target) if target.reblog?
-      account_status_url(target.account, target)
+      if target.reblog?
+        activity_account_status_url(target.account, target)
+      else
+        account_status_url(target.account, target)
+      end
     when :emoji
       emoji_url(target)
+    when :conversation
+      context_url(target)
     end
   end
 
@@ -66,7 +74,9 @@ class ActivityPub::TagManager
       [COLLECTIONS[:public]]
     when 'unlisted', 'private'
       [account_followers_url(status.account)]
-    when 'direct', 'limited'
+    when 'limited'
+      status.conversation_id.present? ? [uri_for(status.conversation)] : []
+    when 'direct'
       if status.account.silenced?
         # Only notify followers if the account is locally silenced
         account_ids = status.active_mentions.pluck(:account_id)
@@ -104,7 +114,7 @@ class ActivityPub::TagManager
       cc << COLLECTIONS[:public]
     end
 
-    unless status.direct_visibility? || status.limited_visibility?
+    unless status.direct_visibility?
       if status.account.silenced?
         # Only notify followers if the account is locally silenced
         account_ids = status.active_mentions.pluck(:account_id)
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index 4dfaea889..873600b0d 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -3,18 +3,44 @@
 #
 # Table name: conversations
 #
-#  id         :bigint(8)        not null, primary key
-#  uri        :string
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
+#  id                :bigint(8)        not null, primary key
+#  uri               :string
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  parent_status_id  :bigint(8)
+#  parent_account_id :bigint(8)
+#  inbox_url         :string
 #
 
 class Conversation < ApplicationRecord
   validates :uri, uniqueness: true, if: :uri?
 
-  has_many :statuses
+  belongs_to :parent_status, class_name: 'Status', optional: true, inverse_of: :conversation
+  belongs_to :parent_account, class_name: 'Account', optional: true
+
+  has_many :statuses, inverse_of: :conversation
+
+  scope :local, -> { where(uri: nil) }
+
+  before_validation :set_parent_account, on: :create
+
+  after_create :set_conversation_on_parent_status
 
   def local?
     uri.nil?
   end
+
+  def object_type
+    :conversation
+  end
+
+  private
+
+  def set_parent_account
+    self.parent_account = parent_status.account if parent_status.present?
+  end
+
+  def set_conversation_on_parent_status
+    parent_status.update_column(:conversation_id, id) if parent_status.present?
+  end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 71596ec2f..28ae80e09 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -50,9 +50,11 @@ class Status < ApplicationRecord
 
   belongs_to :account, inverse_of: :statuses
   belongs_to :in_reply_to_account, foreign_key: 'in_reply_to_account_id', class_name: 'Account', optional: true
-  belongs_to :conversation, optional: true
+  belongs_to :conversation, optional: true, inverse_of: :statuses
   belongs_to :preloadable_poll, class_name: 'Poll', foreign_key: 'poll_id', optional: true
 
+  has_one :owned_conversation, class_name: 'Conversation', foreign_key: 'parent_status_id', inverse_of: :parent_status
+
   belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies, optional: true
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
@@ -63,6 +65,7 @@ class Status < ApplicationRecord
   has_many :mentions, dependent: :destroy, inverse_of: :status
   has_many :active_mentions, -> { active }, class_name: 'Mention', inverse_of: :status
   has_many :media_attachments, dependent: :nullify
+  has_many :capability_tokens, class_name: 'StatusCapabilityToken', inverse_of: :status, dependent: :destroy
 
   has_and_belongs_to_many :tags
   has_and_belongs_to_many :preview_cards
@@ -205,7 +208,9 @@ class Status < ApplicationRecord
     public_visibility? || unlisted_visibility?
   end
 
-  alias sign? distributable?
+  def sign?
+    distributable? || limited_visibility?
+  end
 
   def with_media?
     media_attachments.any?
@@ -264,11 +269,11 @@ class Status < ApplicationRecord
 
   around_create Mastodon::Snowflake::Callbacks
 
-  before_validation :prepare_contents, if: :local?
-  before_validation :set_reblog
-  before_validation :set_visibility
-  before_validation :set_conversation
-  before_validation :set_local
+  before_validation :prepare_contents, on: :create, if: :local?
+  before_validation :set_reblog, on: :create
+  before_validation :set_visibility, on: :create
+  before_validation :set_conversation, on: :create
+  before_validation :set_local, on: :create
 
   after_create :set_poll_id
 
@@ -464,7 +469,7 @@ class Status < ApplicationRecord
       self.in_reply_to_account_id = carried_over_reply_to_account_id
       self.conversation_id        = thread.conversation_id if conversation_id.nil?
     elsif conversation_id.nil?
-      self.conversation = Conversation.new
+      build_owned_conversation
     end
   end
 
diff --git a/app/models/status_capability_token.rb b/app/models/status_capability_token.rb
new file mode 100644
index 000000000..1613569de
--- /dev/null
+++ b/app/models/status_capability_token.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: status_capability_tokens
+#
+#  id         :bigint(8)        not null, primary key
+#  status_id  :bigint(8)
+#  token      :string
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+class StatusCapabilityToken < ApplicationRecord
+  belongs_to :status
+
+  validates :token, presence: true
+
+  before_validation :generate_token, on: :create
+
+  private
+
+  def generate_token
+    self.token = Doorkeeper::OAuth::Helpers::UniqueToken.generate
+  end
+end
diff --git a/app/presenters/activitypub/activity_presenter.rb b/app/presenters/activitypub/activity_presenter.rb
index 5d174767f..1198375b4 100644
--- a/app/presenters/activitypub/activity_presenter.rb
+++ b/app/presenters/activitypub/activity_presenter.rb
@@ -20,6 +20,8 @@ class ActivityPub::ActivityPresenter < ActiveModelSerializers::Model
             else
               ActivityPub::TagManager.instance.uri_for(status.proper)
             end
+          elsif status.limited_visibility?
+            "bear:?#{{ u: ActivityPub::TagManager.instance.uri_for(status.proper), t: status.capability_tokens.first.token }.to_query}"
           else
             status.proper
           end
diff --git a/app/serializers/activitypub/context_serializer.rb b/app/serializers/activitypub/context_serializer.rb
new file mode 100644
index 000000000..99ef9a73b
--- /dev/null
+++ b/app/serializers/activitypub/context_serializer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class ActivityPub::ContextSerializer < ActivityPub::Serializer
+  include RoutingHelper
+
+  attributes :id, :type, :inbox
+
+  def id
+    ActivityPub::TagManager.instance.uri_for(object)
+  end
+
+  def type
+    'Group'
+  end
+
+  def inbox
+    account_inbox_url(object.parent_account)
+  end
+end
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index f26fd93a4..b0e87efe1 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -7,7 +7,7 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
              :in_reply_to, :published, :url,
              :attributed_to, :to, :cc, :sensitive,
              :atom_uri, :in_reply_to_atom_uri,
-             :conversation
+             :conversation, :context
 
   attribute :content
   attribute :content_map, if: :language?
@@ -121,6 +121,12 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     end
   end
 
+  def context
+    return if object.conversation.nil?
+
+    ActivityPub::TagManager.instance.uri_for(object.conversation)
+  end
+
   def local?
     object.account.local?
   end
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 0a383d6a3..f44a02849 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -52,6 +52,7 @@ class PostStatusService < BaseService
     @text         = @options.delete(:spoiler_text) if @text.blank? && @options[:spoiler_text].present?
     @visibility   = @options[:visibility] || @account.user&.setting_default_privacy
     @visibility   = :unlisted if @visibility&.to_sym == :public && @account.silenced?
+    @visibility   = :limited if @visibility&.to_sym != :direct && @in_reply_to&.limited_visibility?
     @scheduled_at = @options[:scheduled_at]&.to_datetime
     @scheduled_at = nil if scheduled_in_the_past?
   rescue ArgumentError
@@ -64,10 +65,11 @@ class PostStatusService < BaseService
 
     ApplicationRecord.transaction do
       @status = @account.statuses.create!(status_attributes)
+      @status.capability_tokens.create! if @status.limited_visibility?
     end
 
-    process_hashtags_service.call(@status)
-    process_mentions_service.call(@status)
+    ProcessHashtagsService.new.call(@status)
+    ProcessMentionsService.new.call(@status)
   end
 
   def schedule_status!
@@ -109,14 +111,6 @@ class PostStatusService < BaseService
     ISO_639.find(str)&.alpha2
   end
 
-  def process_mentions_service
-    ProcessMentionsService.new
-  end
-
-  def process_hashtags_service
-    ProcessHashtagsService.new
-  end
-
   def scheduled?
     @scheduled_at.present?
   end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 79af3fc54..3dd46edf9 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -42,9 +42,21 @@ class ProcessMentionsService < BaseService
       "@#{mentioned_account.acct}"
     end
 
+    if status.limited_visibility? && status.thread&.limited_visibility?
+      # If we are replying to a local status, then we'll have the complete
+      # audience copied here, both local and remote. If we are replying
+      # to a remote status, only local audience will be copied. Then we
+      # need to send our reply to the remote author's inbox for distribution
+
+      status.thread.mentions.includes(:account).find_each do |mention|
+        status.mentions.create(silent: true, account: mention.account)
+      end
+    end
+
     status.save!
     check_for_spam(status)
 
+    # Silent mentions need to be delivered separately
     mentions.each { |mention| create_notification(mention) }
   end
 
diff --git a/app/workers/activitypub/distribution_worker.rb b/app/workers/activitypub/distribution_worker.rb
index e4997ba0e..3f6d7408a 100644
--- a/app/workers/activitypub/distribution_worker.rb
+++ b/app/workers/activitypub/distribution_worker.rb
@@ -12,8 +12,10 @@ class ActivityPub::DistributionWorker
 
     return if skip_distribution?
 
-    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
-      [payload, @account.id, inbox_url]
+    if delegate_distribution?
+      deliver_to_parent!
+    else
+      deliver_to_inboxes!
     end
 
     relay! if relayable?
@@ -24,22 +26,44 @@ class ActivityPub::DistributionWorker
   private
 
   def skip_distribution?
-    @status.direct_visibility? || @status.limited_visibility?
+    @status.direct_visibility?
+  end
+
+  def delegate_distribution?
+    @status.limited_visibility? && @status.reply? && !@status.conversation.local?
   end
 
   def relayable?
     @status.public_visibility?
   end
 
+  def deliver_to_parent!
+    return if @status.conversation.inbox_url.blank?
+
+    ActivityPub::DeliveryWorker.perform_async(payload, @account.id, @status.conversation.inbox_url)
+  end
+
+  def deliver_to_inboxes!
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [payload, @account.id, inbox_url]
+    end
+  end
+
   def inboxes
-    # Deliver the status to all followers.
-    # If the status is a reply to another local status, also forward it to that
-    # status' authors' followers.
-    @inboxes ||= if @status.reply? && @status.thread.account.local? && @status.distributable?
-                   @account.followers.or(@status.thread.account.followers).inboxes
-                 else
-                   @account.followers.inboxes
-                 end
+    # Deliver the status to all followers. If the status is a reply
+    # to another local status, also forward it to that status'
+    # authors' followers. If the status has limited visibility,
+    # deliver it to inboxes of people mentioned (no shared ones)
+
+    @inboxes ||= begin
+      if @status.limited_visibility?
+        DeliveryFailureTracker.without_unavailable(Account.remote.joins(:mentions).merge(@status.mentions).pluck(:inbox_url))
+      elsif @status.reply? && @status.thread.account.local? && @status.distributable?
+        @account.followers.or(@status.thread.account.followers).inboxes
+      else
+        @account.followers.inboxes
+      end
+    end
   end
 
   def payload
diff --git a/app/workers/activitypub/forward_distribution_worker.rb b/app/workers/activitypub/forward_distribution_worker.rb
new file mode 100644
index 000000000..994da978b
--- /dev/null
+++ b/app/workers/activitypub/forward_distribution_worker.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class ActivityPub::ForwardDistributionWorker < ActivityPub::DistributionWorker
+  include Sidekiq::Worker
+
+  sidekiq_options queue: 'push'
+
+  def perform(conversation_id, json)
+    conversation = Conversation.find(conversation_id)
+
+    @status  = conversation.parent_status
+    @account = conversation.parent_account
+    @json    = json
+
+    return if @status.nil? || @account.nil?
+
+    deliver_to_inboxes!
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def payload
+    @json
+  end
+end