OD-2292.1 - reformatted to use parameterized queries in steps 03 and 04

01-Load-Automated-Archive-into-Mysql.py

@@ -1,6 +1,8 @@
+#!/usr/bin/env python3
+
+from automated_archive import aa
 from shared_python.Args import Args
 from shared_python.Sql import Sql
-from automated_archive import aa
 
 if __name__ == "__main__":
     args_obj = Args()
@@ -10,8 +12,6 @@
 
     # eg: python 01-Load-Automated-Archive-into-Mysql.py -dh localhost -du root -dt dsa -dd temp_python -a AA -f /path/to/ARCHIVE_DB.pl -o .
     log.info(
-        'Loading Automated Archive file "{0}" into database "{1}"'.format(
-            args.db_input_file, args.temp_db_database
-        )
+        f'Loading Automated Archive file "{args.db_input_file}" into database "{args.temp_db_database}"'
     )
     aa.clean_and_load_data(args, log)

02a-Load-Chapters-to-Working-Table.py

@@ -1,4 +1,5 @@
-# encoding: utf-8
+#!/usr/bin/env python3
+
 from pymysql import cursors
 
 from shared_python.Args import Args
@@ -8,7 +9,7 @@
 
 # Given an existing final chapter table, this will use the URL field and chapter location to load the chapter contents
 def __current_table(table_name, db):
-    query = "SELECT * FROM `{0}`.`{1}`".format(args.output_database, table_name)
+    query = f"SELECT * FROM `{args.output_database}`.`{table_name}`"
     dict_cursor = db.cursor(cursors.DictCursor)
     dict_cursor.execute(query)
     return dict_cursor.fetchall()
@@ -22,5 +23,5 @@ def __current_table(table_name, db):
     sql = Sql(args, log)
     chaps = Chapters(args, sql, log)
 
-    log.info("Loading chapters from {0}...".format(args.chapters_path))
+    log.info(f"Loading chapters from {args.chapters_path}...")
     chaps.populate_chapters()

02b-Extract-Tags-From-Stories.py

@@ -1,10 +1,11 @@
+#!/usr/bin/env python3
+
 import re
 
 from shared_python.Args import Args
 from shared_python.Sql import Sql
 from shared_python.Tags import Tags
 
-
 if __name__ == "__main__":
     """
   Only for non-eFiction archives.
@@ -18,9 +19,7 @@
     sql = Sql(args, log)
     tags = Tags(args, sql, log)
     log.info(
-        "Processing tags from stories and bookmarks table in {0}".format(
-            args.temp_db_database
-        )
+        f"Processing tags from stories and bookmarks table in {args.temp_db_database}"
     )
 
     tag_col_list = {}

03-Export-Tags-Authors-Stories.py

@@ -1,8 +1,10 @@
+#!/usr/bin/env python3
+
 import csv
-from shared_python.Args import Args
-from shared_python.Sql import Sql
 import html
 
+from shared_python.Args import Args
+from shared_python.Sql import Sql
 from shared_python.Tags import Tags
 
 
@@ -39,7 +41,7 @@ def write_csv(data, filename, columns):
     results = tags.distinct_tags(args.temp_db_database)
     write_csv(
         results,
-        "{0}/{1} - tags.csv".format(args.output_folder, args.archive_name),
+        f"{args.output_folder}/{args.archive_name} - tags.csv",
         [
             cols["id"],
             cols["original_tag"],
@@ -73,28 +75,16 @@ def write_csv(data, filename, columns):
 
     results = sql.execute_and_fetchall(
         args.temp_db_database,
-        """
-    SELECT s.{0} as "Story ID", s.title as "Title", s.summary as "Summary", a.{1} as "Creator", a.email as "Creator Email",
+        f"""
+    SELECT s.{story_id} as "Story ID", s.title as "Title", s.summary as "Summary", a.{author_name} as "Creator", a.email as "Creator Email",
     "" as "New Email address", "" as "AO3 Account? (& does email match?)", "" as "Searched/Found", "" as "Work on AO3?",
     "" as "Import status", "" as "importer/inviter", "" as "import/invite date", "" as "AO3 link", "" as "Notes (if any)"
-    FROM {2} ia join {3} a on ia.{4} = a.{5} join {6} s on ia.{7} = s.{8} where ia.item_type = "story";
-    """.format(
-            story_id,
-            author_name,
-            item_authors_table,
-            author_table,
-            ia_author_col,
-            author_id,
-            stories_table,
-            ia_item_col,
-            story_id,
-        ),
+    FROM {item_authors_table} ia join {author_table} a on ia.{ia_author_col} = a.{author_id} join {stories_table} s on ia.{ia_item_col} = s.{story_id} where ia.item_type = "story";
+    """,
     )
     write_csv(
         results,
-        "{0}/{1} - authors with stories.csv".format(
-            args.output_folder, args.archive_name
-        ),
+        f"{args.output_folder}/{args.archive_name} - authors with stories.csv",
         [
             "Story ID",
             "Title",
@@ -117,9 +107,9 @@ def write_csv(data, filename, columns):
     log.debug(
         f"Exporting authors with bookmarks from {args.temp_db_database} to {args.output_folder}"
     )
-    author_table = "{0}.authors".format(args.temp_db_database)
-    bookmarks_table = "{0}.story_links".format(args.temp_db_database)
-    item_authors_table = "{0}.item_authors".format(args.temp_db_database)
+    author_table = f"{args.temp_db_database}.authors"
+    bookmarks_table = f"{args.temp_db_database}.story_links"
+    item_authors_table = f"{args.temp_db_database}.item_authors"
     author_name = "name"
     bookmark_id = "id"
     bookmark_author_col = "author_id"
@@ -130,29 +120,17 @@ def write_csv(data, filename, columns):
 
     results = sql.execute_and_fetchall(
         args.temp_db_database,
-        """
-      SELECT s.{0} as "Bookmark ID", s.title as "Title", s.summary as "Summary", a.{1} as "Creator", a.email as "Creator Email",
+        f"""
+      SELECT s.{bookmark_id} as "Bookmark ID", s.title as "Title", s.summary as "Summary", a.{author_name} as "Creator", a.email as "Creator Email",
       s.url as "URL",
       "" as "New Email address", "" as "AO3 Account? (& does email match?)", "" as "Searched/Found", "" as "Work on AO3?",
       "" as "Import status", "" as "importer/inviter", "" as "import/invite date", "" as "AO3 link", "" as "Notes (if any)"
-      FROM {2} ia join {3} a on ia.{4} = a.{5} join {6} s on ia.{7} = s.{8} where ia.item_type = "story_link";
-    """.format(
-            bookmark_id,
-            author_name,
-            item_authors_table,
-            author_table,
-            ia_author_col,
-            author_id,
-            bookmarks_table,
-            ia_item_col,
-            bookmark_id,
-        ),
+      FROM {item_authors_table} ia join {author_table} a on ia.{ia_author_col} = a.{author_id} join {bookmarks_table} s on ia.{ia_item_col} = s.{bookmark_id} where ia.item_type = "story_link";
+    """,
     )
     write_csv(
         results,
-        "{0}/{1} - authors with bookmarks.csv".format(
-            args.output_folder, args.archive_name
-        ),
+        f"{args.output_folder}/{args.archive_name} - authors with bookmarks.csv",
         [
             "Bookmark ID",
             "Title",

04-Rename-Tags.py

@@ -1,12 +1,12 @@
-# encoding: utf-8
+#!/usr/bin/env python3
+
 import csv
 
 from shared_python.Args import Args
 from shared_python.Common import print_progress
 from shared_python.Sql import Sql
 from shared_python.Tags import Tags
 
-
 if __name__ == "__main__":
     """
   When Tag Wrangling have finished mapping the tags in Google Drive, export the spreadsheet as a CSV file. This script

05-Create-Open-Doors-Tables.py

@@ -1,4 +1,5 @@
-# encoding: utf-8
+#!/usr/bin/env python3
+
 import os
 
 from shared_python.Args import Args
@@ -15,7 +16,7 @@ def _clean_email(author):
     email = author["email"]
     if email is None or email == "":
         email = (
-            "{0}{1}Archive@ao3.org".format(author["name"], args.archive_name)
+            "{}{}Archive@ao3.org".format(author["name"], args.archive_name)
             .replace(" ", "")
             .replace("'", "")
         )
@@ -28,7 +29,7 @@ def main(args, log):
     sql = Sql(args, log)
     final = FinalTables(args, sql, log)
 
-    log.info("Creating final destination tables in {0}".format(args.output_database))
+    log.info(f"Creating final destination tables in {args.output_database}")
 
     table_names = {
         "authors": "authors",
@@ -50,7 +51,7 @@ def main(args, log):
     if os.path.exists(args.story_ids_to_remove):
         with open(args.story_ids_to_remove, "rt") as f:
             log.info(
-                "Removing {0} Do Not Import stories...".format(
+                "Removing {} Do Not Import stories...".format(
                     sum(line.count(",") for line in f) + 1
                 )
             )
@@ -63,7 +64,7 @@ def main(args, log):
     if args.bookmark_ids_to_remove and os.path.exists(args.bookmark_ids_to_remove):
         with open(args.bookmark_ids_to_remove, "rt") as f:
             log.info(
-                "Removing {0} Do Not Import bookmarks...".format(
+                "Removing {} Do Not Import bookmarks...".format(
                     sum(line.count(",") for line in f) + 1
                 )
             )
@@ -75,25 +76,19 @@ def main(args, log):
     stories_without_tags = final.original_table(
         table_names["stories"], story_exclusion_filter
     )
-    log.info(
-        "Stories without tags after removing DNI: {0}".format(len(stories_without_tags))
-    )
+    log.info(f"Stories without tags after removing DNI: {len(stories_without_tags)}")
     bookmarks_without_tags = final.original_table(
         table_names["story_links"], bookmark_exclusion_filter
     )
     if bookmarks_without_tags:
         log.info(
-            "Bookmarks without tags after removing DNI: {0}".format(
-                len(bookmarks_without_tags)
-            )
+            f"Bookmarks without tags after removing DNI: {len(bookmarks_without_tags)}"
         )
     else:
         log.info("No bookmarks to remove")
 
     # STORIES
-    log.info(
-        "Copying stories to final table {0}.stories...".format(args.output_database)
-    )
+    log.info(f"Copying stories to final table {args.output_database}.stories...")
     final_stories = []
     for story in stories_without_tags:
         story_authors = final.original_table(
@@ -113,9 +108,7 @@ def main(args, log):
     # BOOKMARKS
     if bookmarks_without_tags is not None:
         log.info(
-            "Copying bookmarks to final table {0}.story_links...".format(
-                args.output_database
-            )
+            f"Copying bookmarks to final table {args.output_database}.story_links..."
         )
         final_bookmarks = []
         for bookmark in bookmarks_without_tags:
@@ -132,9 +125,7 @@ def main(args, log):
 
     # AUTHORS
     log.info(
-        "Copying authors to final table {0}.authors, cleaning emails and removing authors with no works...".format(
-            args.output_database
-        )
+        f"Copying authors to final table {args.output_database}.authors, cleaning emails and removing authors with no works..."
     )
     final_authors = []
     authors = final.original_table(table_names["authors"])
@@ -155,30 +146,24 @@ def main(args, log):
     if chapters:
         dest_chapter_table = f"{args.output_database}.{table_names['chapters']}"
         log.info(
-            "Copying chapters table {0} from source chapters table...".format(
-                dest_chapter_table
-            )
+            f"Copying chapters table {dest_chapter_table} from source chapters table..."
         )
-        sql.execute("drop table if exists {0}".format(dest_chapter_table))
+        sql.execute(f"drop table if exists {dest_chapter_table}")
 
         truncate_and_insert = (
-            "create table {0} (unique(id), key(story_id)) select * from {1}.{2}".format(
+            "create table {} (unique(id), key(story_id)) select * from {}.{}".format(
                 dest_chapter_table, args.temp_db_database, table_names["chapters"]
             )
         )
         sql.execute(truncate_and_insert)
 
         add_auto_increment = (
-            "alter table {0} modify id int not null auto_increment".format(
-                dest_chapter_table
-            )
+            f"alter table {dest_chapter_table} modify id int not null auto_increment"
         )
         sql.execute(add_auto_increment)
     else:
         log.info(
-            "Creating chapters table {0}.chapters from source stories table...".format(
-                args.output_database
-            )
+            f"Creating chapters table {args.output_database}.chapters from source stories table..."
         )
         final_chapters = final.dummy_chapters(final_stories)
         final.insert_into_final("chapters", final_chapters)

06-Update-Tags-In-Story-Table.py

@@ -1,11 +1,11 @@
-# encoding: utf-8
+#!/usr/bin/env python3
+
 from shared_python.Args import Args
 from shared_python.FinalTables import FinalTables
 from shared_python.PopulateTags import PopulateTags
 from shared_python.Sql import Sql
 from shared_python.Tags import Tags
 
-
 if __name__ == "__main__":
     """
   Denormalize tags out of the working tags table into comma-separated lists in the stories or story_link tables

08-Check-ODAP-Tables.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import sys
+
 from shared_python.Args import Args
 from shared_python.Sql import Sql