Page MenuHomeDevCentral
Files F11719957

D3678.id9521.diff
No OneTemporary
Actions

D3678.id9521.diff
View Options

diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,9 @@
__pycache__/
*.egg-info/
tools/*/dist/
+
+# pywikibot
+tools/rhyne-wyse/logs/
+tools/rhyne-wyse/apicache/
+*.lwp
+throttle.ctrl
diff --git a/tools/rhyne-wyse/bin/update-agora-reports b/tools/rhyne-wyse/bin/update-agora-reports
new file mode 100755
--- /dev/null
+++ b/tools/rhyne-wyse/bin/update-agora-reports
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+# -------------------------------------------------------------
+# Rhyne-Wise
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: MediaWiki automated agent
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+import logging
+import sys
+
+from rhyne_wyse import client, config
+from rhyne_wyse.tasks.wiki import update_report
+
+
+# -------------------------------------------------------------
+# Application entry point
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def run():
+ global_config = config.get_config()
+ site = client.connect_to_site(global_config["wiki"])
+
+ for report in global_config.get("reports", []):
+ try:
+ update_report(site, logger, report)
+ except Exception as e:
+ title = report.get("report", "<report title missing>")
+ logger.exception(f"[update_report] for report {title}: {e}")
+
+
+if __name__ == "__main__":
+ logger = logging.getLogger(__name__)
+ logger.addHandler(logging.StreamHandler(sys.stderr))
+ logger.setLevel(logging.INFO)
+
+ run()
diff --git a/tools/rhyne-wyse/conf/rhyne-wyse.yaml b/tools/rhyne-wyse/conf/rhyne-wyse.yaml
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/conf/rhyne-wyse.yaml
@@ -0,0 +1,22 @@
+wiki:
+ credentials:
+ driver: vault
+ secret: agora
+
+reports:
+ - report: devcentral-token-language-models
+ tool: fetch
+ tool_options:
+ url: https://docker-002.nasqueron.org/reports/devcentral-tokens-language-models.txt
+ page: AI content
+ tweaks:
+ - compute-hash-ignoring-date
+
+ - report: agora-operations-grimoire-older-pages
+ tool: nasqueron-reports
+ tool_options:
+ vault_credentials: /usr/local/etc/secrets/rhyne-wise.yaml
+ page: Operations grimoire/Old content report
+ tweaks:
+ - compute-hash-first-column
+ - update-at-least-monthly
diff --git a/tools/rhyne-wyse/families/agora_family.py b/tools/rhyne-wyse/families/agora_family.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/families/agora_family.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+
+# -------------------------------------------------------------
+# Rhyne-Wise
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Pywikibot configuration
+# License: BSD-2-Clause
+# Site: Nasqueron Agora
+# -------------------------------------------------------------
+
+
+from pywikibot import family
+
+
+class Family(family.Family): # noqa: D101
+
+ name = "agora"
+ langs = {
+ "agora": "agora.nasqueron.org",
+ }
+
+ def scriptpath(self, code):
+ return {
+ "agora": "",
+ }[code]
+
+ def protocol(self, code):
+ return {
+ "agora": "https",
+ }[code]
diff --git a/tools/rhyne-wyse/requirements.txt b/tools/rhyne-wyse/requirements.txt
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/requirements.txt
@@ -0,0 +1,2 @@
+nasqueron-reports~=0.1.0
+pywikibot~=10.4.0
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/__init__.py b/tools/rhyne-wyse/src/rhyne_wyse/__init__.py
new file mode 100644
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/client.py b/tools/rhyne-wyse/src/rhyne_wyse/client.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/client.py
@@ -0,0 +1,48 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Client
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: MediaWiki client
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from typing import Dict
+
+import pywikibot
+from pywikibot.login import ClientLoginManager
+
+from rhyne_wyse.credentials import vault
+
+
+# -------------------------------------------------------------
+# Wiki authentication
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def fetch_credentials(credentials_config: Dict) -> Dict:
+ try:
+ driver = credentials_config["driver"]
+ except KeyError:
+ raise ValueError("Missing config key: wiki.credentials.driver")
+
+ if driver == "vault":
+ client = vault.connect_to_vault()
+ return vault.read_app_secret(client, credentials_config["secret"])
+
+ raise ValueError(f"Unknown credentials driver: {driver}")
+
+
+def connect_to_site(config: Dict):
+ site = pywikibot.Site()
+
+ credentials = fetch_credentials(config["credentials"])
+ manager = ClientLoginManager(
+ site=site,
+ user=credentials["username"],
+ password=credentials["password"],
+ )
+ manager.login()
+ site.login()
+
+ return site
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/config.py b/tools/rhyne-wyse/src/rhyne_wyse/config.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/config.py
@@ -0,0 +1,27 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Config
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Parse YAML configuration
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+import yaml
+
+
+DEFAULT_CONFIG_PATH = "conf/rhyne-wyse.yaml"
+DEFAULT_HASHES_PATH = "/var/db/rhyne-wyse/hashes"
+
+
+def get_config_path():
+ return DEFAULT_CONFIG_PATH
+
+
+def get_config():
+ with open(get_config_path()) as config_file:
+ return yaml.load(config_file, Loader=yaml.Loader)
+
+
+def get_hashes_path():
+ return DEFAULT_HASHES_PATH
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/credentials/vault.py b/tools/rhyne-wyse/src/rhyne_wyse/credentials/vault.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/credentials/vault.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+# -------------------------------------------------------------
+# Rhyne-Wise :: Credentials :: Vault
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Fetch credentials from Vault
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from typing import Dict
+
+import hvac
+
+
+VAULT_CA_CERTIFICATE = "/usr/local/share/certs/nasqueron-vault-ca.crt"
+
+
+def connect_to_vault():
+ return hvac.Client(
+ verify=VAULT_CA_CERTIFICATE,
+ )
+
+
+def read_secret(
+ vault_client, mount_point: str, prefix: str, key: str
+) -> Dict[str, str]:
+ secret = vault_client.secrets.kv.read_secret_version(
+ mount_point=mount_point,
+ path=prefix + "/" + key,
+ )
+ return secret["data"]["data"]
+
+
+def read_app_secret(vault_client, key: str) -> Dict[str, str]:
+ return read_secret(vault_client, "apps", "rhyne-wyse", key)
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/tasks/reports.py b/tools/rhyne-wyse/src/rhyne_wyse/tasks/reports.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/tasks/reports.py
@@ -0,0 +1,73 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Tasks :: Reports
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from typing import Dict, Tuple
+
+import requests
+
+from nasqueron_reports.actions.reports import generate_report
+from nasqueron_reports.config import parse_report_config
+
+from rhyne_wyse.wiki.page import get_page_age
+from rhyne_wyse.utils.hashes import *
+
+
+# -------------------------------------------------------------
+# Main tasks
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def prepare_report(report_options: Dict) -> Report:
+ if report_options["tool"] == "nasqueron-reports":
+ report_config = parse_report_config(report_options["report"])
+ return generate_report(report_config)
+ elif report_options["tool"] == "fetch":
+ return fetch_report(report_options)
+
+ raise ValueError("Unknown report tool: " + report_options["tool"])
+
+
+def needs_report_update(site, page_title, report: Report, tweaks: List) -> bool:
+ to_update = False
+ report_hash = ""
+
+ # Do not eagerly return True, as we need to update the hash too
+
+ if "update-at-least-monthly" in tweaks:
+ age = get_page_age(site, page_title)
+ if age > 30:
+ to_update = True
+
+ if "compute-hash-ignoring-date" in tweaks:
+ report_hash = compute_hash_ignoring_date(report)
+ elif "compute-hash-first-column" in tweaks:
+ report_hash = compute_hash_from_first_column(report)
+
+ if report_hash is not None:
+ current_hash = read_hash_from_datastore(page_title)
+ if current_hash != report_hash:
+ to_update = True
+ write_hash_to_datastore(page_title, report_hash)
+
+ return to_update
+
+
+# -------------------------------------------------------------
+# Fetch an already generated report from a specific URL
+#
+# For reports configured with `tool: fetch`
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def fetch_report(report_options) -> Report:
+ url = report_options["tool_options"]["url"]
+
+ response = requests.get(url)
+ response.raise_for_status()
+
+ return Report(None, response.text)
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/tasks/wiki.py b/tools/rhyne-wyse/src/rhyne_wyse/tasks/wiki.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/tasks/wiki.py
@@ -0,0 +1,36 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Tasks :: Wiki
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from typing import Dict
+
+import pywikibot
+
+from rhyne_wyse.tasks.reports import prepare_report, needs_report_update
+from rhyne_wyse.wiki.page import update_text_with_new_report
+
+
+def publish_report(site, title, content, comment):
+ page = pywikibot.Page(site, title)
+ page.text = update_text_with_new_report(page.text, content)
+ page.save(summary=comment, minor=False, bot=True)
+
+
+def update_report(site, logger, report_options: Dict):
+ report = prepare_report(report_options)
+
+ tweaks = report_options.get("tweaks", [])
+
+ if needs_report_update(site, report_options["page"], report, tweaks):
+ comment = "[wiki.update_report] Update report " + report_options["report"]
+
+ logger.info(comment)
+ publish_report(site, report_options["page"], report.formatted, comment)
+ else:
+ logger.info(
+ f"[wiki.update_report] Report {report_options['report']} is up to date."
+ )
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/utils/hashes.py b/tools/rhyne-wyse/src/rhyne_wyse/utils/hashes.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/utils/hashes.py
@@ -0,0 +1,89 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Utilities :: Hashes
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+import hashlib
+import re
+from pathlib import Path
+from typing import List, Dict
+
+import yaml
+
+from nasqueron_reports import Report
+from nasqueron_reports.formats.mediawiki import read_as_str
+
+from rhyne_wyse.config import get_hashes_path
+
+# -------------------------------------------------------------
+# Compute hashes
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+REPORT_DATE_RE = r"\d{4}-\d{2}-\d{2} report"
+
+
+def compute_hash_ignoring_date(report: Report) -> str:
+ if report.raw.rows is None:
+ content = [
+ line
+ for line in report.formatted.splitlines()
+ if not re.search(REPORT_DATE_RE, line)
+ ]
+ return compute_hash_from(content)
+ else:
+ return compute_hash_from(report.raw.rows)
+
+
+def compute_hash_from_first_column(report: Report) -> str:
+ if report.raw.rows is None:
+ raise ValueError(
+ "compute-hash-first-column is not supported when raw report is unavailable"
+ )
+
+ content = [read_as_str(row[0]) for row in report.raw.rows]
+ return compute_hash_from(content)
+
+
+def compute_hash_from(content: List[str]) -> str:
+ """Compute SHA-256 hash from a list of strings."""
+ h = hashlib.sha256()
+
+ for line in content:
+ h.update(line.encode("utf-8"))
+ h.update(b"\0") # separator to avoid accidental concatenation collisions
+
+ return h.hexdigest()
+
+
+# -------------------------------------------------------------
+# Hashes datastore
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def key_to_filename(key: str) -> Path:
+ """Convert wiki page title to safe filename for hash storage."""
+ safe = key.replace("/", "__").replace(" ", "_")
+
+ return Path(get_hashes_path(), f"{safe}.sha256")
+
+
+def read_hash_from_datastore(key: str) -> str | None:
+ """Read hash for a given key from its file."""
+ path = key_to_filename(key)
+
+ if not path.exists():
+ return None
+
+ return path.read_text(encoding="utf-8").strip()
+
+
+def write_hash_to_datastore(key: str, hash_to_write: str):
+ """Write hash for a given key to its file."""
+ path = key_to_filename(key)
+
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(hash_to_write + "\n", encoding="utf-8")
diff --git a/tools/rhyne-wyse/src/rhyne_wyse/wiki/page.py b/tools/rhyne-wyse/src/rhyne_wyse/wiki/page.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/src/rhyne_wyse/wiki/page.py
@@ -0,0 +1,87 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Wiki :: Page
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Helper functions to interact with pages on the wiki
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from enum import Enum
+import time
+
+import pywikibot
+
+
+# -------------------------------------------------------------
+# Page metadata
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+def get_page_age(site, page_title: str) -> int:
+ """Get the age of a page from the site, expressed in days."""
+ page = pywikibot.Page(site, page_title)
+
+ unix_timestamp = page.latest_revision.timestamp.posix_timestamp()
+ age_seconds = time.time() - unix_timestamp
+ age_days = age_seconds / 86400
+
+ return int(age_days)
+
+
+# -------------------------------------------------------------
+# Reports
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+REPORT_START = "<!-- Report start -->"
+REPORT_END = "<!-- Report end -->"
+
+
+class Position(Enum):
+ BEFORE_MARKER = 1
+ INSIDE_MARKERS = 2
+ AFTER_MARKER = 3
+
+
+def update_text_with_new_report(current_text: str, report: str) -> str:
+ return replace_between_markers(current_text, REPORT_START, REPORT_END, report)
+
+
+def replace_between_markers(
+ page_text: str, marker_start: str, marker_end: str, new_inner_text: str
+) -> str:
+ """
+ Replace the text between marker_start and marker_end (markers themselves stay)
+ Returns the new page text. If markers not found, raises ValueError.
+ """
+ position = Position.BEFORE_MARKER
+ new_lines = []
+
+ for line in page_text.splitlines():
+ if position == Position.BEFORE_MARKER:
+ new_lines.append(line)
+
+ if marker_start in line:
+ position = Position.INSIDE_MARKERS
+ new_lines.append(new_inner_text)
+ new_lines.append(marker_end)
+
+ continue
+
+ if position == Position.INSIDE_MARKERS:
+ if marker_end in line:
+ position = Position.AFTER_MARKER
+
+ continue
+
+ if position == Position.AFTER_MARKER:
+ new_lines.append(line)
+
+ if position == Position.BEFORE_MARKER:
+ raise ValueError(f"Opening marker not found: {marker_start}")
+
+ if position == Position.INSIDE_MARKERS:
+ raise ValueError(f"Closing marker not found: {marker_end}")
+
+ return "\n".join(new_lines) + "\n" # EOL at EOF
diff --git a/tools/rhyne-wyse/tests/wiki/page.py b/tools/rhyne-wyse/tests/wiki/page.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/tests/wiki/page.py
@@ -0,0 +1,49 @@
+# -------------------------------------------------------------
+# Rhyne-Wise :: Tests :: Wiki :: Page
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+import unittest
+
+from rhyne_wyse.wiki.page import replace_between_markers
+
+
+class PageTest(unittest.TestCase):
+ def test_replace_between_markers(self):
+ current = """The tree is:
+[TREE]
+Fir
+[/TREE]
+"""
+
+ expected = """The tree is:
+[TREE]
+Abies Electronicus
+[/TREE]
+"""
+
+ actual = replace_between_markers(
+ current,
+ "[TREE]",
+ "[/TREE]",
+ "Abies Electronicus",
+ )
+
+ self.assertEqual(expected, actual)
+
+ def test_replace_between_markers_when_missing(self):
+ self.assertRaises(
+ ValueError,
+ replace_between_markers,
+ "The tree is: Fir",
+ "[TREE]",
+ "[/TREE]",
+ "Abies Electronicus",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/rhyne-wyse/user-config.py b/tools/rhyne-wyse/user-config.py
new file mode 100644
--- /dev/null
+++ b/tools/rhyne-wyse/user-config.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+# -------------------------------------------------------------
+# Rhyne-Wise
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# Project: Nasqueron
+# Description: Pywikibot configuration
+# License: BSD-2-Clause
+# -------------------------------------------------------------
+
+
+from pywikibot.config import usernames
+
+
+# -------------------------------------------------------------
+# General configuration
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+
+family = "agora"
+mylang = "agora"
+usernames["agora"]["agora"] = "Rhyne-Wyse"

File Metadata

Mime Type
text/plain
Expires
Tue, Sep 16, 09:35 (14 h, 58 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
2988472
Default Alt Text
D3678.id9521.diff (18 KB)

Event Timeline

Nasqueron DevCentral · If it had been much bigger the moon would have had a core of ice. · Powered by Phabricator