Repopulating the Taaalk database from archive.org

class Taaalk < ApplicationRecord
  belongs_to :user 
  # the user who created the Taaalk

  has_many :speakers
  # Taaalk speakers != users, this is so an individual
  # user can have different profiles depending on who
  # they are talking to

  has_many :messages

  has_one_attached :image
  # the Taaalk's optional background image
end

class Speaker < ApplicationRecord
  belongs_to :taaalk
  belongs_to :user
  has_many :messages

  has_one_attached :image # profile image

  has_rich_text :biography 
  # a speaker's bio; rich text used to enable links
  # (e.g. robertheaton.com in your bio above) 
end

class Message < ApplicationRecord
  belongs_to :user
  belongs_to :taaalk
  belongs_to :speaker

  has_rich_text :content 
  # the raw input from the rich_text_area I'm typing this
  # message in now. We can probably ignore.
  # e.g. Message.last.content.body.to_s
  # "<div class=\"trix-content\">\n
  #   <div>Hi!</div>\n
  # </div>\n"

  has_rich_text :safe_content
  # the raw content which has gone through my javascript
  # parser which adds "text bubbles" to each block. 
  # safe_content is used to render messages in front end.
  # e.g. Message.last.safe_content.body.to_s
  # "<div class=\"trix-content\">\n
  #   <div class=\"tlk-bubble-holder\"><div class=\"tlk-bubble\">Hi!</div></div>\n
  # </div>\n"
end

class User < ApplicationRecord
  has_many :taaalks
  has_many :speakers
  has_many :messages

  has_one_attached :image

  has_rich_text :biography

  validates :username, presence: true
  validates :email, presence: true
end

• User accounts
  I think all Taaalks (bar one) will not continue. So creating faux users is acceptable. I can always manually alter one in the console afterwards. I we should add an admin message at the base of "imported" Taaalk to let the user know how to get in touch to claim their account.

• Script vs HTTP requests
  Script

• QA instance
  Is this so we don't accidentally explode production? How do we do this, and when?

• Message timestamps
  Let's use the original ones. I think that will add some flavour to this challenge too.

• Profile pictures
  When I visit Taaalk on the wayback machine I don't see the profile images, however I do see the Taaalk background image. Neither are in the wayback download so we might have to give this a miss.

$ ruby html_articles_to_json.rb

Dir.each_child("t") do |d|
  dir = "t/" + d
  puts dir
  Dir.each_child(dir) do |f|
    file = File.open(f)
    file_data = file.read
    puts file_data[0, 30]
  end
end

require 'nokogiri'

taaalks = []

Dir.each_child("t") do |d|
  dir = "t/" + d
  Dir.chdir(dir) do
    taaalk = {}
    data = Nokogiri::HTML(File.open("index.html"))

    taaalk[:title] = data.css('h1')[0].text

    taaalk[:speakers] = []
    data.css('.spkr-info').each do |spkr|
      speaker = {}
      spkr.css('h3 a').each do |s|
        speaker[:profile_path] = s['href']
        speaker[:name] = s.text
      end
      speaker[:twitter_handle] = spkr.css('.twitter-handle').text
      speaker[:bio] = spkr.css('.trix-content').inner_html
      taaalk[:speakers] << speaker
    end

    taaalks << taaalk
  end
end

pp taaalks

# => [...,
{:title=>"Bitcoin Maxima & Other Crypto Things",
  :speakers=>
   [{:profile_path=>"/u/joshua-summers",
     :name=>"Joshua Summers",
     :twitter_handle=>" JoshSummers1234",
     :bio=>
      "\n" +
      "  <div>I'm the founder of Taaalk &acirc;&#156;&#140;&iuml;&cedil;&#143;. Confused by crypto. Hopefully not for long.</div>\n"},
    {:profile_path=>"/u/thomas-hartman",
     :name=>"Thomas Hartman",
     :twitter_handle=>" thomashartman1",
     :bio=>
      "\n" +
      "  <div>Bitcoin investor, wealth manager, and project consultant. <br>blog: <a href=\"https://standardcrypto.wordpress.com/\">https://standardcrypto.wordpress.com/</a>\n" +
      "</div>\n"}]}, 
...]

require 'nokogiri'

taaalks = Dir["t/*/index.html"].map do |path|
  page = Nokogiri::HTML(File.open(path))

  speakers = page.css('.spkr-info').map do |spkr_info|
    info_header = spkr_info.css('h3 a')[0]
    {
      name: info_header.text,
      id: info_header.attr('class').delete("^0-9"),
      side: spkr_info.attr('class').gsub('spkr-info spkr-info-',''),
      profile_path: info_header['href'],
      twitter_handle: spkr_info.css('.twitter-handle').text.strip,
      bio: spkr_info.css('.trix-content').inner_html,
    }
  end

  messages = page.css('.tlk-blob').map do |tlk_blob|
    {
      speaker_id: tlk_blob.css('div[class^="name-spkr-"]')[0].attr('class').delete("^0-9"),
      created_at: tlk_blob.css('div[class="tlk-blob-date"]').text.partition("(")[0].strip,
      message: tlk_blob.search('div[target="spkr-color-"]').children.to_html
    }
  end

  {
    title: page.css('h1')[0].text,
    speakers: speakers,
    messages: messages,
  }
end

# individual message example
{
  :speaker_id=>"129",
  :created_at=>"18:20, 05 Feb 21",
  :message=>
      "\n" +
      "            <div class=\"tlk-blob-msg\">\n" +
      "<div class=\"trix-content\">\n" +
      "  <div class=\"tlk-bubble-holder\"><div class=\"tlk-bubble\">...</div></div>\n" +
      "</div>\n" +
      "\n" +
      "            </div>\n" +
      "          "
}

require 'nokogiri'
require 'json'

taaalks = Dir["t/*/index.html"].map do |path|
  page = Nokogiri::HTML(File.open(path))

  speakers = page.css('.spkr-info').map do |spkr_info|
    info_header = spkr_info.css('h3 a')[0]
    {
      name: info_header.text,
      id: info_header.attr('class').delete("^0-9"),
      side: spkr_info.attr('class').gsub('spkr-info spkr-info-',''),
      profile_path: info_header['href'],
      twitter_handle: spkr_info.css('.twitter-handle').text.strip,
      bio: spkr_info.css('.trix-content').inner_html,
    }
  end

  messages = page.css('.tlk-blob').map do |tlk_blob|
    {
      speaker_id: tlk_blob.css('div[class^="name-spkr-"]')[0].attr('class').delete("^0-9"),
      created_at: tlk_blob.css('div[class="tlk-blob-date"]').text.split("(")[0].strip,
      message: tlk_blob.search('div[target="spkr-color-"]').children.to_html
    }
  end

  {
    title: page.css('h1')[0].text,
    speakers: speakers,
    messages: messages,
  }
end

File.open('taaalks.json', 'w') { |file| file.write(taaalks.to_json) }