Add basic lint setup with ruff

.github/workflows/git-hooks.yml

@@ -0,0 +1,29 @@
+ame: "Git Hooks"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request: ~
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-prek:
+    name: prek run --all-files
+    runs-on: ubuntu-24.04
+    timeout-minutes: 1
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Run prek
+        uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece  # v3.0.0

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.16.2
+    hooks:
+      - id: ruff-check
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format

ruff.toml

@@ -0,0 +1,2 @@
+line-length = 120
+target-version = "py311"

scripts/add_new_contribution_to_yaml.py

@@ -1,29 +1,30 @@
 """
 given properties, add a new contribution to the contributions.yaml database file.
 """
-from datetime import datetime, UTC
+
 import json
 import pathlib
+from datetime import UTC, datetime
 from sys import argv
 
 from ruamel.yaml import YAML
 
 
 def split_categories(categories):
-    categories = sorted(categories.replace('"', '').split(','))
+    categories = sorted(categories.replace('"', "").split(","))
     categories = [category.strip() for category in categories]
     return categories
 
 
 def postprocess_properties(properties_dict):
-    if 'categories' in properties_dict and properties_dict['categories']:
-        properties_dict['categories'] = split_categories(properties_dict['categories'])
+    if properties_dict.get("categories"):
+        properties_dict["categories"] = split_categories(properties_dict["categories"])
     else:
-        properties_dict['categories'] = None
+        properties_dict["categories"] = None
 
     # add download
-    if 'download' not in properties_dict:
-        properties_dict['download'] = properties_dict['source'][:properties_dict['source'].rfind('.')] + '.zip'
+    if "download" not in properties_dict:
+        properties_dict["download"] = properties_dict["source"][: properties_dict["source"].rfind(".")] + ".zip"
 
 
 if __name__ == "__main__":
@@ -35,29 +36,29 @@ def postprocess_properties(properties_dict):
     postprocess_properties(props)
 
     # open database
-    database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml'
+    database_file = pathlib.Path(__file__).parent.parent / "contributions.yaml"
 
     yaml = YAML()
-    with open(database_file, 'r') as db:
+    with open(database_file, "r") as db:
         data = yaml.load(db)
 
-    contributions_list = list(data['contributions'])
+    contributions_list = list(data["contributions"])
 
     # find max index
     max_index = max([int(contribution["id"]) for contribution in contributions_list])
 
     # append new contribution with next index
     # add status, at top
-    datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
+    datetime_today = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S%z")
     contribution = {
-        'id': max_index + 1,
-        'status': 'VALID',
-        'dateAdded': datetime_today,
+        "id": max_index + 1,
+        "status": "VALID",
+        "dateAdded": datetime_today,
     }
     contribution.update(props)
 
     contributions_list.append(contribution)
 
     # write all contributions to database file
-    with open(database_file, 'w') as db:
+    with open(database_file, "w") as db:
         yaml.dump({"contributions": contributions_list}, db)

scripts/fetch_updates.py

@@ -1,124 +1,128 @@
 """
 Reads in the contributions.yaml file, and updates the entries by hitting the 'source' url.
 """
+
 import argparse
-from datetime import datetime, UTC
 import pathlib
-from ruamel.yaml import YAML
+from datetime import UTC, datetime
 from multiprocessing import Pool
 
-from parse_and_validate_properties_txt import read_properties_txt, parse_text, validate_existing
+from parse_and_validate_properties_txt import parse_text, read_properties_txt, validate_existing
+from ruamel.yaml import YAML
 
 
 def update_contribution(contribution, props):
-  datetime_today = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%S%z')
-  contribution['lastUpdated'] = datetime_today
-  if 'previousVersions' not in contribution:
-    contribution['previousVersions'] = []
-  contribution['previousVersions'].append(contribution['prettyVersion'])
-
-  # update from online
-  for field in props.keys():
-    # process category list
-    if field == 'categories':
-      if props[field]:
-        contribution[field] = sorted(props[field].strip('"').split(','))
-      else:
-        contribution[field] = []
-    else:
-      contribution[field] = props[field]
+    datetime_today = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S%z")
+    contribution["lastUpdated"] = datetime_today
+    if "previousVersions" not in contribution:
+        contribution["previousVersions"] = []
+    contribution["previousVersions"].append(contribution["prettyVersion"])
+
+    # update from online
+    for field in props:
+        # process category list
+        if field == "categories":
+            if props[field]:
+                contribution[field] = sorted(props[field].strip('"').split(","))
+            else:
+                contribution[field] = []
+        else:
+            contribution[field] = props[field]
+
+    if "download" not in contribution:
+        contribution["download"] = contribution["source"][: contribution["source"].rfind(".")] + ".zip"
 
-  if 'download' not in contribution:
-    contribution['download'] = contribution['source'][:contribution['source'].rfind('.')] + '.zip'
-
 
 def log_broken(contribution, msg):
-  if contribution['status'] == 'VALID':
-    contribution['status'] = 'BROKEN'
-    if 'log' not in contribution:
-      contribution['log'] = []
-    contribution['log'].append(msg)
-
-def process_contribution(item):
-  index, contribution = item
+    if contribution["status"] == "VALID":
+        contribution["status"] = "BROKEN"
+        if "log" not in contribution:
+            contribution["log"] = []
+        contribution["log"].append(msg)
 
-  date_today = datetime.now(UTC).strftime('%Y-%m-%d')
-  this_version = '0'
 
-  if contribution['status'] != 'DEPRECATED':
-    # compare version to what is at url. If has changed, update contribution to
-    # what is online
-    if 'version' in contribution:
-      this_version = contribution['version']
+def process_contribution(item):
+    index, contribution = item
+
+    date_today = datetime.now(UTC).strftime("%Y-%m-%d")
+    this_version = "0"
+
+    if contribution["status"] != "DEPRECATED":
+        # compare version to what is at url. If has changed, update contribution to
+        # what is online
+        if "version" in contribution:
+            this_version = contribution["version"]
+
+        try:
+            properties_raw = read_properties_txt(contribution["source"])
+        except FileNotFoundError as e:
+            log_broken(contribution, f"file not found, {e}, {date_today}")
+            return index, contribution
+        # TODO: Specify which exceptions are excepted and drop noqa: BLE001
+        except Exception:  # noqa: BLE001
+            log_broken(contribution, f"url timeout, {date_today}")
+            return index, contribution
+
+        try:
+            props = validate_existing(parse_text(properties_raw))
+        # TODO: Specify which exceptions are excepted and drop noqa: BLE001
+        except Exception:  # noqa: BLE001
+            log_broken(contribution, f"invalid file, {date_today}")
+            return index, contribution
+
+        # some library files have field lastUpdated. This also exists in the database, but is defined
+        # by our scripts, so remove this field.
+        contribution.pop("lastUpdated", None)
+
+        contribution["status"] = "VALID"
+
+        if props["version"] != this_version:
+            # update from online
+            update_contribution(contribution, props)
+    return index, contribution
 
-    try:
-      properties_raw = read_properties_txt(contribution['source'])
-    except FileNotFoundError as e:
-      log_broken(contribution, f'file not found, {e}, {date_today}')
-      return index, contribution
-    except Exception:
-      log_broken(contribution, f'url timeout, {date_today}')
-      return index, contribution
 
-    try:
-        props = validate_existing(parse_text(properties_raw))
-    except Exception:
-      log_broken(contribution, f'invalid file, {date_today}')
-      return index, contribution
+def process_all(contributions_list):
+    total = len(contributions_list)
+    completed = 0
+    print(f"Starting processing of {total} contributions...")
 
-    # some library files have field lastUpdated. This also exists in the database, but is defined
-    # by our scripts, so remove this field.
-    contribution.pop('lastUpdated', None)
+    with Pool(processes=256) as pool:
+        for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
+            contributions_list[index] = contribution
+            completed += 1
+            print(f"Progress: {completed}/{total} ({(completed / total * 100):.1f}%)")
 
-    contribution['status'] = 'VALID'
 
-    if props['version'] != this_version:
-      # update from online
-      update_contribution(contribution, props)
-  return index, contribution
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--index")
+    args = parser.parse_args()
 
+    index = "all"
+    if args.index:
+        index = args.index
 
-def process_all(contributions_list):
-  total = len(contributions_list)
-  completed = 0
-  print(f"Starting processing of {total} contributions...")
+    database_file = pathlib.Path(__file__).parent.parent / "contributions.yaml"
 
-  with Pool(processes=256) as pool:
-    for index, contribution in pool.imap_unordered(process_contribution, enumerate(contributions_list)):
-      contributions_list[index] = contribution
-      completed += 1
-      print(f"Progress: {completed}/{total} ({(completed / total * 100):.1f}%)")
+    # read in database yaml file
+    yaml = YAML()
+    with open(database_file, "r") as db:
+        data = yaml.load(db)
 
+    contributions_list = data["contributions"]
 
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--index')
-  args = parser.parse_args()
-
-  index = 'all'
-  if args.index:
-    index = args.index
-
-  database_file = pathlib.Path(__file__).parent.parent / 'contributions.yaml'
-
-  # read in database yaml file
-  yaml = YAML()
-  with open(database_file, 'r') as db:
-    data = yaml.load(db)
-
-  contributions_list = data['contributions']
-
-  if index == 'all':
-    process_all(contributions_list)
-    print("All processing complete")
-  else:
-    # update only contribution with id==index
-    contribution = next((x for x in contributions_list if x['id'] == int(index)), None)
-    print(contribution)
-    process_contribution((index, contribution))
-    print(contribution)
-
-  # write all contributions to database file
-  yaml = YAML()
-  with open(database_file, 'w') as outfile:
-    yaml.dump({"contributions": contributions_list}, outfile)
+    if index == "all":
+        process_all(contributions_list)
+        print("All processing complete")
+    else:
+        # update only contribution with id==index
+        contribution = next((x for x in contributions_list if x["id"] == int(index)), None)
+        print(contribution)
+        process_contribution((index, contribution))
+        print(contribution)
+
+    # write all contributions to database file
+    yaml = YAML()
+    with open(database_file, "w") as outfile:
+        yaml.dump({"contributions": contributions_list}, outfile)