Add support for multiple Netfile Agencies

First steps of #75.

* Remove hardcoding of "COAK" Netfile agency and replace with a table
  that stores agencies
* Create database columns to keep track of which subscribers are
  subscribed to which Netfile agencies
* Create database columns to keep track of which filings came from which
  Netfile agencies
* Update alert mailer to only show subscribers the filings from their
  agency

.gitignore

@@ -39,3 +39,4 @@
 /yarn-error.log
 yarn-debug.log*
 .yarn-integrity
+.idea

app/controllers/alert_subscribers_controller.rb

@@ -61,6 +61,6 @@ def set_alert_subscriber
   end
 
   def alert_subscriber_params
-    params.fetch(:alert_subscriber).permit(:email)
+    params.fetch(:alert_subscriber).permit(:email).merge(netfile_agency: NetfileAgency.coak)
   end
 end

app/lib/disclosure_downloader.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 class DisclosureDownloader
-  def initialize
+  def initialize(agency = NetfileAgency.coak)
     @netfile = Netfile::Client.new
+    @agency = agency
   end
 
   def download
@@ -14,7 +15,7 @@ def download
     puts "Latest: #{latest&.filed_at}"
     puts '==================================================================='
 
-    @netfile.each_filing do |json|
+    @netfile.each_filing(agency: @agency) do |json|
       filing = Filing.from_json(json)
 
       if filing.new_record?

app/lib/disclosure_emailer.rb

@@ -8,24 +8,29 @@ def initialize(date)
   end
 
   def send_email
-    puts '==================================================================='
-    puts 'Emailing:'
-    puts
-    puts "Filings in date range: #{filings_in_date_range.length}"
-    puts '==================================================================='
-    return if filings_in_date_range.none?
+    NetfileAgency.each_supported_agency do |agency|
+      subscribers = AlertSubscriber.subscribed.where(netfile_agency: agency)
+      filings = filings_in_date_range(agency)
 
-    AlertSubscriber.subscribed.find_each do |subscriber|
-      AlertMailer
-        .daily_alert(subscriber, @date, filings_in_date_range, notices_in_date_range)
-        .deliver_now
+      puts '==================================================================='
+      puts "Emailing to #{subscribers.count} subscribers of #{agency.shortcut}:"
+      puts
+      puts "Total filings in date range: #{filings.length}"
+      puts '==================================================================='
+      return if filings.none?
+
+      AlertSubscriber.subscribed.find_each do |subscriber|
+        AlertMailer
+          .daily_alert(subscriber, @date, filings, notices_in_date_range)
+          .deliver_now
+      end
     end
   end
 
   private
 
-  def filings_in_date_range
-    Filing.filed_on_date(@date)
+  def filings_in_date_range(agency)
+    Filing.filed_on_date(@date).where(netfile_agency: agency)
   end
 
   def notices_in_date_range

app/lib/netfile/client.rb

@@ -82,15 +82,15 @@ def get_filing(filing_id)
       end
     end
 
-    def each_filing(form: nil, &block)
-      return to_enum(:each_filing) unless block_given?
+    def each_filing(form: nil, agency:, &block)
+      return to_enum(:each_filing, form: form, agency: agency) unless block_given?
 
       Net::HTTP.start(BASE_URL.host, BASE_URL.port, use_ssl: true) do |http|
         with_pagination do |current_page|
           request = Net::HTTP::Post.new(BASE_URL + 'public/list/filing')
           request['Accept'] = 'application/json'
           request.body = URI.encode_www_form(
-            AID: 'COAK',
+            AID: agency.shortcut,
             CurrentPageIndex: current_page,
             Form: form,
           )

app/mailers/alert_mailer.rb

@@ -7,9 +7,9 @@ class AlertMailer < ApplicationMailer
   track open: true, click: true, utm_params: true,
     user: -> { AlertSubscriber.subscribed.find_by(email: message.to.first) }
 
-  def daily_alert(alert_subscriber, date_or_date_range, filings_in_date_range, notice)
+  def daily_alert(alert_subscriber, date_or_date_range, filings, notice)
     @alert_subscriber = alert_subscriber
-    @forms = Forms.from_filings(filings_in_date_range)
+    @forms = Forms.from_filings(filings)
     @email_notice = notice
 
     subject_date = if date_or_date_range.is_a?(Range)

app/models/alert_subscriber.rb

@@ -18,6 +18,7 @@ class AlertSubscriber < ApplicationRecord
   scope :unsubscribed, -> { where.not(unsubscribed_at: nil) }
 
   has_many :ahoy_messages, foreign_key: :user_id
+  belongs_to :netfile_agency
 
   validates :email, format: /\A[^@]+@[^\.]+\.[\w]+\z/i
 

app/models/filing.rb

@@ -11,6 +11,7 @@ class Filing < ApplicationRecord
   has_many :election_candidates, foreign_key: :fppc_id, primary_key: :filer_id
   has_one :election_committee, foreign_key: :fppc_id, primary_key: :filer_id
   has_one :amended_filing, class_name: 'Filing', primary_key: :amended_filing_id, foreign_key: :id
+  belongs_to :netfile_agency
 
   def election_referendum
     ElectionReferendum
@@ -28,6 +29,7 @@ def self.from_json(json)
       record.filer_id = json['filerStateId']
       record.filer_name = json['filerName']
       record.title = json['title']
+      record.netfile_agency = NetfileAgency.by_netfile_id(json['agency'])
       record.filed_at = DateTime.parse(json['filingDate'])
       record.amendment_sequence_number = json['amendmentSequenceNumber']
       record.amended_filing_id = json['amendedFilingId']

app/models/netfile_agency.rb

@@ -0,0 +1,24 @@
+class NetfileAgency < ApplicationRecord
+  def self.create_supported_agencies
+    find_or_create_by(netfile_id: 13, shortcut: 'COAK', name: 'Oakland, City of')
+    find_or_create_by(netfile_id: 52, shortcut: 'SFO', name: 'San Francisco Ethics Commission')
+  end
+
+  def self.coak
+    @_coak ||= find_by(shortcut: 'COAK')
+  end
+
+  def self.sfo
+    @_sfo ||= find_by(shortcut: 'SFO')
+  end
+
+  def self.each_supported_agency(&block)
+    block.call(coak)
+    block.call(sfo)
+  end
+
+  def self.by_netfile_id(id)
+    @_by_id ||= all.index_by(&:netfile_id)
+    @_by_id.fetch(id)
+  end
+end

db/migrate/20221014195907_add_agency_id_to_filings.rb

@@ -0,0 +1,29 @@
+class AddAgencyIdToFilings < ActiveRecord::Migration[7.0]
+  def up
+    create_table :netfile_agencies do |t|
+      t.integer :netfile_id
+      t.string :shortcut
+      t.string :name
+
+      t.index :netfile_id, unique: true
+      t.index :shortcut, unique: true
+    end
+
+    oakland = NetfileAgency.create(netfile_id: 13, shortcut: 'COAK', name: 'Oakland, City of')
+    _sf = NetfileAgency.create(netfile_id: 52, shortcut: 'SFO', name: 'San Francisco Ethics Commission')
+
+    change_table :filings do |t|
+      t.references :netfile_agency, default: oakland.id
+    end
+
+    change_table :alert_subscribers do |t|
+      t.references :netfile_agency, default: oakland.id
+    end
+  end
+
+  def down
+    remove_reference :netfile_filings, :agency
+    remove_reference :netfile_alert_subscribers, :agency
+    drop_table :netfile_agencies
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_10_09_211246) do
+ActiveRecord::Schema[7.0].define(version: 2022_10_14_195907) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -60,6 +60,8 @@
     t.string "token"
     t.datetime "unsubscribed_at", precision: nil
     t.datetime "confirmed_at", precision: nil
+    t.bigint "netfile_agency_id", default: 1
+    t.index ["netfile_agency_id"], name: "index_alert_subscribers_on_netfile_agency_id"
     t.index ["token"], name: "index_alert_subscribers_on_token"
   end
 
@@ -111,6 +113,16 @@
     t.datetime "filed_at", precision: nil
     t.json "contents"
     t.xml "contents_xml"
+    t.bigint "netfile_agency_id", default: 1
+    t.index ["netfile_agency_id"], name: "index_filings_on_netfile_agency_id"
+  end
+
+  create_table "netfile_agencies", force: :cascade do |t|
+    t.integer "netfile_id"
+    t.string "shortcut"
+    t.string "name"
+    t.index ["netfile_id"], name: "index_netfile_agencies_on_netfile_id", unique: true
+    t.index ["shortcut"], name: "index_netfile_agencies_on_shortcut", unique: true
   end
 
   create_table "notices", force: :cascade do |t|

spec/admin/alert_subscribers_spec.rb

@@ -7,7 +7,8 @@
   let!(:alert_subscriber) do
     AlertSubscriber.create!(
       email: 'subscriber@example.com',
-      confirmed_at: Time.now
+      confirmed_at: Time.now,
+      netfile_agency: NetfileAgency.coak
     )
   end
 

spec/controllers/alert_subscribers_controller_spec.rb

@@ -28,6 +28,7 @@
         expect(subscriber.token).to be_present
         expect(subscriber.confirmed_at).to be_nil
         expect(subscriber.unsubscribed_at).to be_nil
+        expect(subscriber.netfile_agency).to eq(NetfileAgency.coak)
       end
 
       it 'sends a AlertSubscriberMailer.confirm email' do
@@ -53,7 +54,7 @@
   end
 
   describe '#edit' do
-    let(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com') }
+    let(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com', netfile_agency: NetfileAgency.coak) }
     let(:request_token) { nil }
 
     subject { get :edit, params: { id: alert_subscriber.id, token: request_token } }
@@ -72,7 +73,7 @@
   end
 
   describe '#destroy' do
-    let!(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com') }
+    let!(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com', netfile_agency: NetfileAgency.coak) }
     let(:request_token) { nil }
 
     subject { post :destroy, params: { id: alert_subscriber.id, token: request_token } }
@@ -99,7 +100,7 @@
   describe '#confirm' do
     render_views
 
-    let!(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com') }
+    let!(:alert_subscriber) { AlertSubscriber.create(email: 'tomdooner+test@gmail.com', netfile_agency: NetfileAgency.coak) }
     let(:request_token) { nil }
 
     subject { post :confirm, params: { id: alert_subscriber.id, token: request_token } }

spec/controllers/webhooks_controller_spec.rb

@@ -9,7 +9,8 @@
     let(:subscriber) do
       AlertSubscriber.create(
         email: 'test@example.com',
-        confirmed_at: Time.now
+        confirmed_at: Time.now,
+        netfile_agency: NetfileAgency.coak,
       )
     end
     let!(:admin_user) { AdminUser.create(email: 'admin@example.com', password: 'secretpassword') }

spec/helpers/alert_mailer_helper_spec.rb

@@ -79,7 +79,7 @@
         { 'form_Type' => 'F460', 'line_Item' => '5', 'amount_A' => 1000 }, # total contributions received
       ]
     end
-    let(:filing) { Filing.create(filer_id: 123, form: 30, contents: filing_contents) }
+    let(:filing) { Filing.create(filer_id: 123, form: 30, contents: filing_contents, netfile_agency: NetfileAgency.coak) }
     let(:form) { Forms.from_filings([filing]).first }
 
     it 'returns the value when there is no amended form' do
@@ -88,7 +88,7 @@
 
     context 'with an amended form' do
       let(:amended_filing_contents) { filing_contents.dup.tap { |c| c[0]['amount_A'] = 2000 } }
-      let(:filing_amendment) { Filing.create(form: 30, amended_filing_id: filing.id, contents: amended_filing_contents) }
+      let(:filing_amendment) { Filing.create(form: 30, amended_filing_id: filing.id, contents: amended_filing_contents, netfile_agency: NetfileAgency.coak) }
       let(:form_amendment) { Forms.from_filings([filing_amendment]).first }
 
       it 'returns an amended value' do

spec/lib/disclosure_downloader_spec.rb

@@ -6,7 +6,7 @@ def initialize(fake_filings)
       @filings = Array(fake_filings)
     end
 
-    def each_filing(&block)
+    def each_filing(agency:, &block)
       @filings.each { |filing| block.call(filing.metadata) }
     end
 
@@ -31,7 +31,8 @@ def fetch_transaction_contents(id)
         filingDate: "2022-10-12T17:50:07.0000000-07:00",
         amendmentSequenceNumber: 0,
         amendedFilingId: nil,
-        form: 36 # FPPC Form 496
+        form: 36, # FPPC Form 496
+        agency: NetfileAgency.coak.netfile_id,
       }.stringify_keys
     end
     let(:fake_filing_data) do
@@ -70,6 +71,7 @@ def fetch_transaction_contents(id)
 
       last_filing = Filing.last
       expect(last_filing.title).to eq(fake_filing.metadata['title'])
+      expect(last_filing.netfile_agency).to eq(NetfileAgency.coak)
     end
   end
 end

spec/lib/disclosure_emailer_spec.rb

@@ -0,0 +1,57 @@
+require 'rails_helper'
+
+describe DisclosureEmailer do
+  describe '#send_email' do
+    let(:today) { Date.parse('2022-10-01') }
+    let(:subscribed_agency) { NetfileAgency.coak }
+    let(:filing_agency) { NetfileAgency.coak }
+    let!(:subscriber) do
+      AlertSubscriber.create(
+        email: 'test@example.com',
+        confirmed_at: Time.now,
+        netfile_agency: subscribed_agency,
+      )
+    end
+    let(:filed_at) { today.beginning_of_day.change(hour: 10) }
+    let!(:filing) { Filing.create(filer_id: 123, filed_at: filed_at, form: 30, netfile_agency: filing_agency) }
+    let!(:valid_filing) { Filing.create(filer_id: 123, filed_at: today.beginning_of_day.change(hour: 10), form: 30, netfile_agency: subscribed_agency) }
+
+    subject { described_class.new(today).send_email }
+
+    before do
+      allow(AlertMailer).to receive(:daily_alert).and_return(double(deliver_now: nil))
+      allow_any_instance_of(DisclosureEmailer).to receive(:puts)
+    end
+
+    context 'with a filing for the appropriate date and agency' do
+      it 'includes that filing' do
+        subject
+        expect(AlertMailer)
+          .to have_received(:daily_alert)
+          .with(subscriber, today, include(filing), anything)
+      end
+    end
+
+    context 'when the filing is for a different date' do
+      let(:filed_at) { today - 1.day }
+
+      it 'excludes the filing' do
+        subject
+        expect(AlertMailer)
+          .to have_received(:daily_alert)
+          .with(subscriber, today, exclude(filing), anything)
+      end
+    end
+
+    context 'when the filing is for a different agency' do
+      let(:filing_agency) { NetfileAgency.sfo }
+
+      it 'excludes the filing' do
+        subject
+        expect(AlertMailer)
+          .to have_received(:daily_alert)
+          .with(subscriber, today, exclude(filing), anything)
+      end
+    end
+  end
+end

spec/lib/forms_spec.rb

@@ -47,7 +47,7 @@
     end
 
     context 'when one of the forms is amended and the other is not' do
-      let(:original_filing) { Filing.create(form: 36, title: 'Oaklanders for a better Oakland', contents: JSON.parse(<<~JSON)) }
+      let(:original_filing) { Filing.create(form: 36, title: 'Oaklanders for a better Oakland', netfile_agency: NetfileAgency.coak, contents: JSON.parse(<<~JSON)) }
         [
           {"form_Type":"F496P3","tran_Dscr":"","tran_Date":"2020-10-09T00:00:00.0000000-07:00","calculated_Amount":25000.0,"cand_NamL":null,"sup_Opp_Cd":null,"bal_Name":null,"bal_Num":null,"tran_NamL":"Service Employees International Union Local 1021 Candidate PAC","tran_NamF":"","tran_City":"Sacramento","tran_Zip4":"95814","tran_Emp":"","tran_Occ":"","tran_Amt1":25000.0,"tran_Amt2":0.0,"entity_Cd":"SCC","cmte_Id":"1296948"},
           {"form_Type":"F496","tran_Dscr":"PHONE CALLS","tran_Date":"2020-10-09T00:00:00.0000000-07:00","calculated_Amount":5830.5,"cand_NamL":"OtherCandidate","sup_Opp_Cd":"S","bal_Name":"","bal_Num":"","tran_NamL":null,"tran_NamF":null,"tran_City":null,"tran_Zip4":null,"tran_Emp":null,"tran_Occ":null,"tran_Amt1":5830.5,"tran_Amt2":null,"entity_Cd":null,"cmte_Id":null}
@@ -72,7 +72,7 @@
     context 'when one of the 496 forms is an amended version of the other one' do
       let(:filing1) { super().tap(&:save) }
       let(:filing2) { super().tap(&:save) }
-      let(:filing3) { Filing.create(form: 36, title: 'Oaklanders for a better Oakland', contents: JSON.parse(<<~JSON)) }
+      let(:filing3) { Filing.create(form: 36, title: 'Oaklanders for a better Oakland', netfile_agency: NetfileAgency.coak, contents: JSON.parse(<<~JSON)) }
         [
           {"form_Type":"F496P3","tran_Dscr":"","tran_Date":"2020-10-09T00:00:00.0000000-07:00","calculated_Amount":35000.0,"cand_NamL":null,"sup_Opp_Cd":null,"bal_Name":null,"bal_Num":null,"tran_NamL":"Service Employees International Union Local 1021 Candidate PAC","tran_NamF":"","tran_City":"Sacramento","tran_Zip4":"95814","tran_Emp":"","tran_Occ":"","tran_Amt1":25000.0,"tran_Amt2":0.0,"entity_Cd":"SCC","cmte_Id":"1296948"},
           {"form_Type":"F496","tran_Dscr":"PHONE CALLS","tran_Date":"2020-10-09T00:00:00.0000000-07:00","calculated_Amount":5830.5,"cand_NamL":"OtherCandidate","sup_Opp_Cd":"S","bal_Name":"","bal_Num":"","tran_NamL":null,"tran_NamF":null,"tran_City":null,"tran_Zip4":null,"tran_Emp":null,"tran_Occ":null,"tran_Amt1":5830.5,"tran_Amt2":null,"entity_Cd":null,"cmte_Id":null}