added 260907 version of worker toolkit

worker-toolkit-flaredown/repo/backend/lib/tasks/app.rake

@@ -0,0 +1,92 @@
+namespace :app do
+  desc "flaredown | setup application"
+  task setup: :environment do
+    setup
+  end
+
+  desc "flaredown | invite beta user to join in"
+  task invite: :environment do
+    invite
+  end
+
+  desc "flaredown | delete a user's account"
+  task delete_account: :environment do
+    delete_account
+  end
+
+  class OperationAbortedException < StandardError; end
+
+  def invite
+    email = prompt "Please insert email address: "
+    User.invite!(email: email)
+  end
+
+  def setup
+    puts "This will create the necessary stuff. You will lose any previous data stored"
+    # NOTE: asking for confirmation is actually a good idea
+    # as DBs will be purged but it messes with CI atm
+    p "Rails env is:"
+    p Rails.env
+    ask_to_continue unless Rails.env.test?
+
+    build_database
+  rescue OperationAbortedException
+    puts "\nQuitting...".red
+    exit 1
+  end
+
+  def build_database
+    header "build database"
+    # Rake::Task["db:mongoid:purge"].invoke
+
+    if Rails.env.development?
+      Rake::Task["db:drop"].invoke
+      Rake::Task["db:create"].invoke
+      Rake::Task["db:migrate"].invoke
+      ActiveRecord::Base.connection.schema_cache.clear!
+      ActiveRecord::Base.descendants.each(&:reset_column_information)
+    end
+    Rake::Task["db:fixtures:load"].invoke
+    Rake::Task["db:seed"].invoke
+  rescue ::PG::ObjectInUse => e
+    puts "\n#{e.message}.".red
+  end
+
+  def header(message)
+    puts ">>> #{message}".bold.blue
+  end
+
+  def ask_to_continue
+    answer = prompt("Do you want to continue (yes/no)? ", %w[yes no])
+    fail OperationAbortedException unless answer.eql?("yes")
+  end
+
+  def prompt(message, choices = nil)
+    begin
+      print(message)
+      input = STDIN.gets
+      raise OperationAbortedException if input.nil?
+
+      answer = input.chomp
+      p "got answer: #{answer}"
+    end while choices.present? && !choices.include?(answer)
+    answer
+  rescue Interrupt
+    raise OperationAbortedException
+  end
+
+  def delete_account
+    puts "DANGER ZONE: This action can't be undone!".yellow
+    email = prompt "Please insert user's email address: "
+    user = User.find_by(email: email)
+    if user.nil?
+      puts "No user found for the entered email address".red
+    else
+      user.destroy!
+      puts "User account successfully deleted".blue
+    end
+  rescue OperationAbortedException
+    puts "\nQuitting..."
+    exit 1
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/encrypt_user_id.rake

@@ -0,0 +1,17 @@
+namespace :app do
+  desc "flaredown | encrypt user_id"
+  task encrypt_user_id: :environment do
+    puts "Updating #{Checkin.count} checkins:\n"
+
+    Checkin.where(:user_id.exists => true).order_by(date: :desc).each_with_index do |c, i|
+      c.user_id = c[:user_id]
+
+      c.unset(:user_id)
+      c.save
+
+      puts i if i % 100 == 0
+    end
+
+    puts "\nDone."
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/hbi_completeness.rake

@@ -0,0 +1,24 @@
+namespace :app do
+  desc "flaredown | percentage of user completed HBI survey"
+  task hbi_completeness: :environment do
+    HBI_CONDITION = "Crohn's disease".freeze
+
+    since = HarveyBradshawIndex.order_by(date: :asc).first.date
+    hbi_count = HarveyBradshawIndex.count
+    hbi_condition_id = Condition::Translation.find_by(name: HBI_CONDITION).condition_id
+    checkin_ids = Checkin::Condition.where(condition_id: hbi_condition_id).distinct(:checkin_id)
+
+    possible_checkins_count = Checkin.where(:date => (since..Date.today), :id.in => checkin_ids).count do |checkin|
+      checkin.harvey_bradshaw_index.present? || chickin_was_available_for_hbi?(checkin)
+    end
+
+    puts 100.0 * hbi_count / possible_checkins_count
+  end
+
+  def chickin_was_available_for_hbi?(checkin)
+    HarveyBradshawIndex.where(
+      encrypted_user_id: checkin.encrypted_user_id,
+      date: ((checkin.date - Checkin::HBI_PERIODICITY).next...(checkin.date))
+    ).none?
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/notifications_scheduler.rake

@@ -0,0 +1,6 @@
+namespace :notification do
+  desc "Schedule NotificationDispatcher"
+  task dispatcher: :environment do
+    NotificationDispatcher.perform_async
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/oneoff.rake

@@ -0,0 +1,268 @@
+namespace :oneoff do
+  task build_trackable_usages: :environment do
+    build_trackable_usages
+  end
+  def build_trackable_usages
+    TrackableUsage.delete_all
+    User.all.each do |user|
+      Rails.logger.info("[#{__method__}] === #{user.email} ===")
+      user.checkins.each do |checkin|
+        Rails.logger.info("[#{__method__}] --- #{checkin.date.strftime("%Y-%m-%d")} ---")
+        Rails.logger.info("[#{__method__}] --- Conditions ---")
+        checkin.conditions.each do |condition|
+          trackable = Condition.find(condition.condition_id)
+          usage = TrackableUsage.create_or_update_by(user: user, trackable: trackable)
+          Rails.logger.info("[#{__method__}] - #{trackable.name}: #{usage.count}")
+        end
+        Rails.logger.info("[#{__method__}] --- Symptoms ---")
+        checkin.symptoms.each do |symptom|
+          trackable = Symptom.find(symptom.symptom_id)
+          usage = TrackableUsage.create_or_update_by(user: user, trackable: trackable)
+          Rails.logger.info("[#{__method__}] - #{trackable.name}: #{usage.count}")
+        end
+        Rails.logger.info("[#{__method__}] --- Treatments ---")
+        checkin.treatments.each do |treatment|
+          trackable = Treatment.find(treatment.treatment_id)
+          usage = TrackableUsage.create_or_update_by(user: user, trackable: trackable)
+          Rails.logger.info("[#{__method__}] - #{trackable.name}: #{usage.count}")
+        end
+      end
+    end
+  end
+
+  task fix_trackable_colors: :environment do
+    fix_trackable_colors
+  end
+  # FIXME
+  # rubocop:disable Metrics/AbcSize
+  def fix_trackable_colors
+    colors = (0..32).to_a
+    User.all.each do |user|
+      Rails.logger.info("[#{__method__}] === #{user.email} ===")
+      colors_hash = {}
+      user.checkins.each do |checkin|
+        Rails.logger.info("[#{__method__}] --- #{checkin.date.strftime("%Y-%m-%d")} ---")
+        used_colors = []
+        fix_and_store_color_id = lambda do |checkin_trackable, trackable_type|
+          trackable_id = checkin_trackable.send("#{trackable_type.downcase}_id")
+          key = "#{trackable_type}-#{trackable_id}"
+          if colors_hash[key].present?
+            checkin_trackable.update!(color_id: colors_hash[key])
+          else
+            if checkin_trackable.color_id.nil?
+              trackable = trackable_type.constantize.find(trackable_id)
+              checkin_trackable.color_id = Flaredown::Colorable.color_id_for(trackable, user)
+            end
+            if used_colors.include?(checkin_trackable.color_id)
+              checkin_trackable.color_id = if used_colors.count >= colors.count
+                used_colors.count % colors.count
+              else
+                (colors - used_colors).sample
+              end
+            end
+            checkin_trackable.save!
+            colors_hash[key] = checkin_trackable.color_id
+            used_colors << checkin_trackable.color_id
+          end
+        end
+        Rails.logger.info("[#{__method__}] --- Conditions ---")
+        checkin.conditions.each do |condition|
+          fix_and_store_color_id.call(condition, "Condition")
+        end
+        Rails.logger.info("[#{__method__}] --- Symptoms ---")
+        checkin.symptoms.each do |symptom|
+          fix_and_store_color_id.call(symptom, "Symptom")
+        end
+        Rails.logger.info("[#{__method__}] --- Treatments ---")
+        checkin.treatments.each do |treatment|
+          fix_and_store_color_id.call(treatment, "Treatment")
+        end
+      end
+      user.trackings.active_at(Time.now).each do |tracking|
+        key = "#{tracking.trackable_type.downcase}-#{tracking.trackable_id}"
+        tracking.update!(color_id: colors_hash[key]) if colors_hash[key].present?
+      end
+      Rails.logger.info("[#{__method__}] Colors: #{colors_hash}")
+    end
+  end
+  # rubocop:enable Metrics/AbcSize
+
+  task generate_screen_names: :environment do
+    generate_screen_names
+  end
+  def generate_screen_names
+    Profile.all.each do |profile|
+      if profile.screen_name.blank?
+        generated_screen_name = profile.user.email.split("@")[0]
+        profile.update!(screen_name: generated_screen_name)
+      end
+    end
+  end
+
+  task remove_checkin_trackable_duplicates: :environment do
+    remove_checkin_trackable_duplicates
+  end
+  def remove_checkin_trackable_duplicates
+    [Checkin::Condition, Checkin::Symptom, Checkin::Treatment].each do |c|
+      c.all.each do |trackable|
+        key = "#{c.name.demodulize.downcase}_id"
+        count = c.where(
+          :checkin => trackable.checkin,
+          key.to_sym => trackable.send(key)
+        ).count
+        trackable.destroy if count > 1
+      end
+    end
+  end
+
+  task generate_trackable_positions: :environment do
+    Checkin.all.each do |checkin|
+      checkin.conditions.sort_by { |t| t.position.to_i }
+        .each_with_index do |condition, i|
+          condition.update!(position: i)
+        end
+      checkin.symptoms.sort_by { |t| t.position.to_i }
+        .each_with_index do |symptom, i|
+          symptom.update!(position: i)
+        end
+      checkin.treatments.sort_by { |t| t.position.to_i }
+        .each_with_index do |treatment, i|
+          treatment.update!(position: i)
+        end
+    end
+  end
+
+  task save_most_recent_trackables_positions: :environment do
+    Profile.update_all(
+      most_recent_conditions_positions: {},
+      most_recent_symptoms_positions: {},
+      most_recent_treatments_positions: {}
+    )
+    User.all.each do |user|
+      latest_checkin = user.checkins.min(date: -1)
+      next if latest_checkin.nil?
+      latest_checkin.conditions.each_with_index do |cc, i|
+        condition = Condition.find(cc.condition_id)
+        user.profile.set_most_recent_condition_position(condition, cc.position || i)
+      end
+      latest_checkin.symptoms.each_with_index do |cs, i|
+        symptom = Symptom.find(cs.symptom_id)
+        user.profile.set_most_recent_symptom_position(symptom, cs.position || i)
+      end
+      latest_checkin.treatments.each_with_index do |ct, i|
+        treatment = Treatment.find(ct.treatment_id)
+        user.profile.set_most_recent_treatment_position(treatment, ct.position || i)
+      end
+      user.profile.save!
+    end
+  end
+
+  task update_last_commented_for_posts: :environment do
+    Post.where(_type: "Post").each do |post|
+      post.update(last_commented: post.created_at)
+    end
+  end
+
+  task add_position_reference_to_checkins_and_weathers: :environment do
+    add_position_reference_to_checkins
+    add_position_reference_to_weathers
+  end
+
+  def add_position_reference_to_checkins
+    Checkin.all.each do |checkin|
+      postal_code = checkin.postal_code
+      next if postal_code.nil? || postal_code.blank?
+
+      PositionReferenceJob.perform_async("Checkin", checkin.id.to_s, postal_code)
+    end
+  end
+
+  def add_position_reference_to_weathers
+    Weather.find_each(batch_size: 500) do |weather|
+      postal_code = weather.postal_code
+      next if postal_code.nil? || postal_code.blank?
+
+      PositionReferenceJob.perform_async("Weather", weather.id, postal_code)
+    end
+  end
+
+  task :send_optional_email, [:email] => :environment do |t, args|
+    email = args[:email]
+    abort("Email should be present") unless email
+
+    CheckinReminderMailer.remind(email: email).deliver_now
+  end
+
+  task :send_top_post_email, [:email] => :environment do |t, args|
+    notify_token = User.find_by(email: args[:email])&.profile&.notify_token
+    abort("There is no such email") if notify_token.nil?
+
+    GroupTopPostsJob.perform_async(notify_token)
+  end
+
+  # Send email with posts, comments, reactions count
+  task :send_notification_email, [:email] => :environment do |t, args|
+    email = args[:email]
+    abort("Email should be present") unless email
+
+    NotificationsMailer.notify(email: email, data: {}).deliver_now
+  end
+
+  # Send email with statistic about Promotion Rate for given time
+  # end_promotion_rate_statistic['email@ex.com','17-09-2017','22-09-2017']
+  task :send_promotion_rate_statistic, [:email, :start_time, :end_time] => :environment do |t, args|
+    email = args[:email]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+
+    objects = PromotionRate.group_by_score_and_date(start_time, end_time)
+
+    PromotionRate::StatisticMailer.show(email, objects.to_a, start_time, end_time).deliver_later
+  end
+
+  # send_promotion_rate_low_rate['email@ex.com','17-09-2017','22-09-2017']
+  task :send_promotion_rate_low_rate, [:email, :start_time, :end_time] => :environment do |t, args|
+    email = args[:email]
+    start_time = args[:start_time]
+    end_time = args[:end_time]
+
+    objects = PromotionRate.filter_low_scores_by_date(start_time, end_time)
+    serialized_objects = objects.to_a.map { |obj| obj.each_with_object({}) { |(k, v), h| h[k] = v.to_s; } }
+
+    PromotionRate::LowRateMailer.show(email, serialized_objects.to_a, start_time, end_time).deliver_later
+  end
+
+  task :update_checkin_reminder, ["profile_id"] => :environment do |t, args|
+    profile_id = args[:profile_id]
+
+    abort UpdateCheckinReminders.perform_async(profile_id) if profile_id
+
+    Profile.where(checkin_reminder: true).find_each do |profile|
+      UpdateCheckinReminders.perform_async(profile.id)
+    end
+  end
+
+  # remove_users_by_email['email1 email2']
+  task :remove_users_by_email, ["email_list"] => :environment do |t, args|
+    email_string = args["email_list"]
+    abort if email_string.blank?
+
+    email_array = email_string.split(" ")
+    email_array.each do |email|
+      user = User.find_by(email: email)
+      next if user.blank?
+
+      remove_user(user, email)
+    end
+  end
+
+  def remove_user(user, email)
+    ActiveRecord::Base.transaction do
+      user.profile&.destroy!
+      user.checkins.or({:postal_code.ne => nil}, :position_id.ne => nil)
+                    &.update_all(postal_code: nil, position_id: nil)
+
+      user.destroy!
+    end
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/rubocop.rake

@@ -0,0 +1,4 @@
+if Rails.env.development?
+  require "rubocop/rake_task"
+  RuboCop::RakeTask.new
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/spec.rake

@@ -0,0 +1,28 @@
+namespace :spec do
+  begin
+    require "rspec/core/rake_task"
+
+    Rake::Task[:system].clear
+    RSpec::Core::RakeTask.new(:system) do |t|
+      t.rspec_opts = "--tag type:system"
+    end
+  rescue LoadError
+    # no rspec available
+  end
+
+  namespace :system do
+    task :start_frontend do
+      require_relative "../../spec/system/support/frontend_app"
+
+      frontend_app = SystemSpec::FrontendApp.instance
+      frontend_app.start $stdout
+
+      trap "INT" do
+        frontend_app.stop
+        exit 1
+      end
+
+      frontend_app.wait_on
+    end
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/trackables.rake

@@ -0,0 +1,23 @@
+namespace :trackables do
+  desc "flaredown | merge trackables"
+
+  TRACKABLE_TYPES = %w[condition symptom treatment tag].freeze
+
+  task :merge, [:trackable_type, :translation] => :environment do |t, args|
+    trackable_type = args[:trackable_type]
+    translation = args[:translation]
+
+    abort MergeTrackables::Dispatcher.perform_async(trackable_type, translation) if trackable_type.present?
+
+    TRACKABLE_TYPES.map { |trackable_type| MergeTrackables::Dispatcher.perform_async(trackable_type) }
+  end
+
+  task :list_same_trackabels, [:trackable_type, :translation] => :environment do |t, args|
+    trackable_type = args[:trackable_type]
+    translation = args[:translation]
+
+    trackable_class = trackable_type.capitalize.constantize
+
+    SameTrackablesJob.perform_async(trackable_type: trackable_type, translation: translation)
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/usda.rake

@@ -0,0 +1,39 @@
+namespace :usda do
+  FOODS_FILE = Rails.root.join("spec/fixtures/usda/FOOD_DES.txt")
+  SURROUNDER = /\A~|~\z/
+  BLANK_STRING = ""
+  PROGRESS_FAIL = "x"
+  PROGRESS_SUCCESS = "."
+
+  desc "Load USDA data"
+  task load: :environment do
+    puts "Loading foods:\n"
+
+    I18n.locale = :en
+
+    FOODS_FILE.readlines.each do |food|
+      begin
+        parsed_food = food.encode("UTF-8", invalid: :replace).split("^")
+
+        created_food = Food.find_or_create_by(ndb_no: strip(parsed_food.first)) do |f|
+          f.comname = strip(parsed_food[4])
+          f.sciname = strip(parsed_food[9])
+          f.long_desc = strip(parsed_food[2])
+          f.shrt_desc = strip(parsed_food[3])
+        end
+      rescue => e
+        puts food
+
+        raise e
+      end
+
+      printf created_food.persisted? ? PROGRESS_SUCCESS : PROGRESS_FAIL
+    end
+
+    puts "\nDone."
+  end
+
+  def strip(str)
+    str.gsub(SURROUNDER, BLANK_STRING)
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/utils.rake

@@ -0,0 +1,98 @@
+require "csv"
+require "ruby-progressbar"
+
+namespace :utils do
+  task csv_export: :environment do
+    headers = %w[
+      user_id age sex country checkin_date
+      trackable_id trackable_type trackable_name trackable_value
+    ]
+
+    progress_bar = ProgressBar.create(
+      title: "Checkins",
+      total: Checkin.count,
+      format: "%t: |%B| %a / %E"
+    )
+
+    CSV.open("export.csv", "wb") do |csv|
+      csv << headers
+
+      User.all.each do |user|
+        age = user.profile.age
+        sex = user.profile.sex_id
+        country = user.profile.country_id
+
+        user.checkins.each do |checkin|
+          progress_bar.increment
+
+          user_id = SymmetricEncryption.encrypt(user.id)
+          checkin_date = checkin.date.strftime("%Y-%m-%d")
+
+          %w[condition symptom treatment].each do |trackable_type|
+            trackable_type_plural = trackable_type.pluralize
+            trackable_type_capital = trackable_type.capitalize
+            trackable_translation_class = "#{trackable_type_capital}::Translation".constantize
+            checkin_trackables = checkin.send(trackable_type_plural)
+
+            checkin_trackables.each do |checkin_trackable|
+              next if checkin_trackable.value.nil?
+
+              trackable_id = checkin_trackable.send("#{trackable_type}_id")
+              trackable_translation = trackable_translation_class.find_by(id: trackable_id)
+              next if trackable_translation.blank?
+
+              csv << [
+                user_id, age, sex, country, checkin_date,
+                trackable_id, trackable_type_capital,
+                trackable_translation.name, checkin_trackable.value
+              ]
+            end
+          end
+
+          checkin.tag_ids.each do |tag_id|
+            tag = Tag::Translation.find_by(tag_id: tag_id)
+
+            csv << [
+              user_id, age, sex, country, checkin_date,
+              tag_id, "Tag", tag.name, nil
+            ]
+          end
+
+          checkin.food_ids.each do |food_id|
+            food = Food::Translation.find_by(food_id: food_id)
+
+            csv << [
+              user_id, age, sex, country, checkin_date,
+              food_id, "Food", food.long_desc, nil
+            ]
+          end
+
+          harvey_bradshaw_index = checkin.harvey_bradshaw_index
+          if harvey_bradshaw_index.present?
+            csv << [
+              user_id, age, sex, country, checkin_date,
+              harvey_bradshaw_index.id, "HBI", "HBI", harvey_bradshaw_index.score
+            ]
+          end
+
+          weather_id = checkin.weather_id
+
+          next if weather_id.blank?
+
+          weather = Weather.find(weather_id)
+
+          %w[icon temperature_min temperature_max precip_intensity pressure humidity].each do |measure|
+            csv << [
+              user_id, age, sex, country, checkin_date,
+              weather_id, "Weather", measure, weather[measure]
+            ]
+          end
+        end
+      end
+    end
+
+    progress_bar.finish
+
+    puts "All done."
+  end
+end

worker-toolkit-flaredown/repo/backend/lib/tasks/weekly_top_posts_scheduler.rake

@@ -0,0 +1,6 @@
+namespace :weekly_top_posts do
+  desc "Schedule TopPostsMailerDispatcher"
+  task dispatcher: :environment do
+    TopPostsMailerDispatcher.perform_async
+  end
+end