From d7dc025af6d80bd050f63136919650a69f0c5a3a Mon Sep 17 00:00:00 2001 From: Alpha Chen Date: Sun, 12 Nov 2023 19:48:14 -0800 Subject: [PATCH] rebuild pinboard data --- db/migrations/002_pinboard_archive.rb | 14 ++++++-- ...{004_create_tags.rb => 003_create_tags.rb} | 4 +-- ...003_enforce_unique_entities_on_pinboard.rb | 7 ---- ...{005_create_urls.rb => 004_create_urls.rb} | 2 +- ...tes.rb => 005_create_sources_and_notes.rb} | 6 ++-- .../006_extract_pinboard_title_notes.rb | 32 +++++++++++++++++++ scripts/20231111_archive_pinboard.rb | 15 --------- 7 files changed, 50 insertions(+), 30 deletions(-) rename db/migrations/{004_create_tags.rb => 003_create_tags.rb} (91%) delete mode 100644 db/migrations/003_enforce_unique_entities_on_pinboard.rb rename db/migrations/{005_create_urls.rb => 004_create_urls.rb} (90%) rename db/migrations/{006_create_sources_and_notes.rb => 005_create_sources_and_notes.rb} (68%) create mode 100644 db/migrations/006_extract_pinboard_title_notes.rb delete mode 100644 scripts/20231111_archive_pinboard.rb diff --git a/db/migrations/002_pinboard_archive.rb b/db/migrations/002_pinboard_archive.rb index 1b366ea..7b8c0fd 100644 --- a/db/migrations/002_pinboard_archive.rb +++ b/db/migrations/002_pinboard_archive.rb @@ -1,13 +1,23 @@ +pinboard_export = File.expand_path(ENV.fetch("PINBOARD_EXPORT")) + Sequel.migration do - change do + up do create_table(:pinboard) do primary_key :id - foreign_key :entity_id, :entities + foreign_key :entity_id, :entities, null: false, unique: true String :json, text: true, null: false DateTime :created_at, null: false DateTime :modified_at, null: false end + + File.foreach(pinboard_export).with_index do |line, i| + created_at = modified_at = DateTime.now + json = line.chomp + + entity_id = DB[:entities].insert(created_at:, modified_at:) + DB[:pinboard].insert(entity_id:, json: , created_at:, modified_at:) + end end end diff --git a/db/migrations/004_create_tags.rb b/db/migrations/003_create_tags.rb similarity index 91% rename from db/migrations/004_create_tags.rb rename to db/migrations/003_create_tags.rb index 2915204..db3f7c5 100644 --- a/db/migrations/004_create_tags.rb +++ b/db/migrations/003_create_tags.rb @@ -10,8 +10,8 @@ Sequel.migration do end create_table(:entities_tags) do - foreign_key :entity_id, :entities - foreign_key :tag_id, :tags + foreign_key :entity_id, :entities, null: false + foreign_key :tag_id, :tags, null: false DateTime :created_at, null: false DateTime :modified_at, null: false diff --git a/db/migrations/003_enforce_unique_entities_on_pinboard.rb b/db/migrations/003_enforce_unique_entities_on_pinboard.rb deleted file mode 100644 index d1c3a13..0000000 --- a/db/migrations/003_enforce_unique_entities_on_pinboard.rb +++ /dev/null @@ -1,7 +0,0 @@ -Sequel.migration do - change do - alter_table(:pinboard) do - add_unique_constraint(:entity_id) - end - end -end diff --git a/db/migrations/005_create_urls.rb b/db/migrations/004_create_urls.rb similarity index 90% rename from db/migrations/005_create_urls.rb rename to db/migrations/004_create_urls.rb index 0709617..90d01af 100644 --- a/db/migrations/005_create_urls.rb +++ b/db/migrations/004_create_urls.rb @@ -2,7 +2,7 @@ Sequel.migration do up do create_table(:urls) do primary_key :id - foreign_key :entity_id, :entities, unique: true + foreign_key :entity_id, :entities, null: false, unique: true String :url, unique: true, null: false diff --git a/db/migrations/006_create_sources_and_notes.rb b/db/migrations/005_create_sources_and_notes.rb similarity index 68% rename from db/migrations/006_create_sources_and_notes.rb rename to db/migrations/005_create_sources_and_notes.rb index 25181e3..2fcc8db 100644 --- a/db/migrations/006_create_sources_and_notes.rb +++ b/db/migrations/005_create_sources_and_notes.rb @@ -1,8 +1,8 @@ Sequel.migration do change do create_table(:sources) do - foreign_key :entity_id, :entities - foreign_key :source_id, :entities + foreign_key :entity_id, :entities, null: false + foreign_key :source_id, :entities, null: false DateTime :created_at, null: false DateTime :modified_at, null: false @@ -11,7 +11,7 @@ Sequel.migration do end create_table(:notes) do - foreign_key :entity_id, :entities + foreign_key :entity_id, :entities, null: false String :text, text: true diff --git a/db/migrations/006_extract_pinboard_title_notes.rb b/db/migrations/006_extract_pinboard_title_notes.rb new file mode 100644 index 0000000..fd54206 --- /dev/null +++ b/db/migrations/006_extract_pinboard_title_notes.rb @@ -0,0 +1,32 @@ +Sequel.migration do + up do + create_table(:titles) do + foreign_key :entity_id, :entities, null: false, unique: true + + String :title + + DateTime :created_at, null: false + DateTime :modified_at, null: false + end + + json = Sequel.sqlite_json_op(:json) + description = json.extract("$.description") + extended = json.extract("$.extended") + + DB[:pinboard] + .select_append(description.as(:description), extended.as(:extended)) + .each do |pin| + entity_id = pin.fetch(:entity_id) + title = pin.fetch(:description) + text = pin.fetch(:extended) + created_at = modified_at = DateTime.now + + DB[:titles].insert(entity_id:, title:, created_at:, modified_at:) + DB[:notes].insert(entity_id:, text:, created_at:, modified_at:) + end + end + + down do + drop_table(:titles) + end +end diff --git a/scripts/20231111_archive_pinboard.rb b/scripts/20231111_archive_pinboard.rb deleted file mode 100644 index 512dde4..0000000 --- a/scripts/20231111_archive_pinboard.rb +++ /dev/null @@ -1,15 +0,0 @@ -require "json" - -require_relative "../db" - -entities = DB[:entities] -pinboard = DB[:pinboard] - -path = File.expand_path("~/Downloads/pinboard_export.2023.11.11_23.05.json_nd") -File.foreach(path).with_index do |line, i| - created_at = modified_at = DateTime.now - json = line.chomp - - entity_id = entities.insert(created_at:, modified_at:) - pinboard.insert(entity_id:, json: , created_at:, modified_at:) -end