indicomb2 0.1__py3-none-any.whl

indicomb2/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1"

indicomb2/contribution.py

@@ -0,0 +1,27 @@
+import datetime
+from dataclasses import dataclass
+
+
+@dataclass
+class Contribution:
+    title: str
+    url: str
+    speakers: list[dict] | str
+    minutes: str
+    start_time: str = None  # HH:MM:SS
+
+    @classmethod
+    def from_indico(cls, start_time: str, speakers: list[dict], **kwargs):
+        start_time = datetime.datetime.strptime(start_time, "%H:%M:%S").replace(
+            tzinfo=datetime.timezone.utc
+        )
+        speakers = ", ".join(
+            [f"{s['first_name']} {s['last_name']}" for s in speakers],
+        )
+        return cls(speakers=speakers, start_time=start_time, **kwargs)
+
+    def __str__(self):
+        return f" - [{self.title}]({self.url}) - {self.speakers}"
+
+    def __lt__(self, other):
+        return self.start_time < other.start_time

indicomb2/event.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass
+
+from indicomb2.contribution import Contribution
+from indicomb2.markdown import Markdown
+
+
+@dataclass
+class Event:
+    url: str
+    date: str
+    title: str
+    contributions: list[dict] | list[Contribution]
+
+    @classmethod
+    def from_indico(cls, url: str, date: str, title: str, contributions: list[dict]):
+        contributions = sorted(
+            [
+                Contribution.from_indico(
+                    title=c["title"],
+                    url=c["url"],
+                    speakers=c["speakers"],
+                    minutes=c.get("note"),
+                    start_time=c["startDate"]["time"],
+                )
+                for c in reversed(contributions)
+            ],
+        )
+        return cls(url, date, title, contributions)
+
+    @classmethod
+    def from_json(cls, url: str, date: str, title: str, contributions: list[Contribution]):
+        contributions = [Contribution(**c) for c in contributions]
+        return cls(url, date, title, contributions)
+
+    def __lt__(self, other):
+        return self.date > other.date
+
+    def __str__(self):
+        md = Markdown()
+        date = md.link(self.date, self.url)
+        md += md.header(f"{date} - {self.title}", level=4)
+        md += "\n".join([str(c) for c in self.contributions])
+        return str(md)
+
+    def minutes(self):
+        md = Markdown()
+        date = md.link(self.date, self.url)
+        md += md.header(f"{date} - {self.title}", level=4)
+        for c in self.contributions:
+            if c.minutes:
+                md += md.admonition("note", str(c)[3:], str(c.minutes["html"]))
+        return str(md)

indicomb2/indicomb2.py

@@ -0,0 +1,182 @@
+import argparse
+import datetime
+import json
+import logging
+import multiprocessing as mp
+import os
+import urllib
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+import requests
+import yaml
+
+from indicomb2.event import Event
+from indicomb2.meetings import Meetings
+
+logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
+
+
+class BearerAuth(requests.auth.AuthBase):
+    def __init__(self, token: str):
+        self.token = token
+
+    def __call__(self, req):
+        req.headers["authorization"] = "Bearer " + self.token
+        return req
+
+
+@dataclass
+class Indico:
+    category_id: int
+    api_token: str = None
+    site: str = "https://indico.cern.ch"
+    years: int = 1
+
+    def __post_init__(self):
+        if self.api_token is None:
+            self.api_token = os.environ.get("INDICO_API_TOKEN", None)
+        if self.api_token is None:
+            msg = (
+                "Please provide an API token or set INDICO_API_TOKEN or INDICO_API_KEY in your env"
+            )
+            raise ValueError(msg)
+
+        self.url = f"{self.site}/category/{self.category_id}/"
+        today = datetime.datetime.now(tz=datetime.timezone.utc).date()
+        start_date = today - datetime.timedelta(days=365 * self.years)
+        self.start_date = start_date.strftime("%Y-%m-%d")
+        logging.info(
+            f"Feteching meetings for {self.category_id} from {self.start_date}",
+        )
+        self.events = self.get_category(
+            category_id=self.category_id,
+            start_date=self.start_date,
+        )
+
+    def request(self, path: str, params: dict):
+        items = list(params.items()) if hasattr(params, "items") else list(params)
+        items = sorted(items, key=lambda x: x[0].lower())
+        url = f"{self.site}{path}?{urllib.parse.urlencode(items)}"
+        response = requests.get(url, auth=BearerAuth(self.api_token), timeout=30)
+        response.encoding = response.apparent_encoding
+        response.raise_for_status()
+        return response.json()
+
+    def get_category(self, category_id: int, start_date: str, end_date: str = "+15d"):
+        category_params = {"from": start_date, "to": end_date, "order": "start"}
+        category_path = f"/export/categ/{category_id}.json"
+        return self.request(category_path, params=category_params)["results"]
+
+    @staticmethod
+    def filter_events(
+        results: dict,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ):
+        if include is None:
+            include = []
+        if exclude is None:
+            exclude = []
+        include = [x.lower() for x in include]
+        exclude = [x.lower() for x in exclude]
+        return [
+            r
+            for r in results
+            if any(x in r["title"].lower() for x in include)
+            and not any(x in r["title"].lower() for x in exclude)
+        ]
+
+    def get_contributions(self, event: dict, params: dict):
+        event_path = f"/export/event/{event['id']}.json"
+        event = self.request(event_path, params=params)["results"]
+        assert len(event) == 1
+        event = event[0]
+        return Event.from_indico(
+            url=event["url"],
+            date=event["startDate"]["date"],
+            title=event["title"],
+            contributions=event["contributions"],
+        )
+
+    def get_meetings(
+        self,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> Meetings:
+        if exclude is None:
+            exclude = []
+        exclude += ["cancelled"]
+        events = self.filter_events(self.events, include=include, exclude=exclude)
+        params = {"detail": "contributions"}
+        results = []
+        pool = mp.Pool(processes=mp.cpu_count())
+        for event in events:
+            kwargs = {"event": event, "params": params}
+            pool.apply_async(self.get_contributions, kwds=kwargs, callback=results.append)
+        pool.close()
+        pool.join()
+        logging.info(f"Fetched {len(results)} events")
+        return Meetings(sorted(results), self.url, self.start_date)
+
+
+def main(args=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-c",
+        "--config",
+        required=True,
+        type=str,
+        help="Path to the config file",
+    )
+    args = parser.parse_args(args)
+
+    # load config
+    with Path(args.config).open() as f:
+        config = yaml.safe_load(f)
+
+    # get sub configs
+    summary_config = config["meeting_summaries"]
+    topical_contributions_config = config["topical_contributions"]
+    minutes_config = config["minutes_summary"]
+
+    # dump meetings to json
+    all_meetings = {}
+    for entry in config["scrape"]:
+        ind = Indico(category_id=entry["category"], years=entry["years"])
+        for name, kwargs in entry["groups"].items():
+            logging.info(f"Fetching meetings for {name}")
+            meetings = ind.get_meetings(**kwargs)
+            all_meetings[name] = meetings
+            Path("json").mkdir(exist_ok=True)
+            fpath = Path(f"json/{name}.json")
+            with fpath.open("w") as f:
+                meetings_dict = [asdict(m) for m in meetings.meetings]
+                json.dump(meetings_dict, f, indent=4, default=str)
+            logging.info(f"Saved to {fpath}")
+
+    # meeting summaries
+    for name, kwargs in summary_config.items():
+        logging.info(f"Creating {name} summary")
+        meetings = all_meetings[name]
+        meetings.meeting_list_md(name=name, **kwargs)
+
+    # add contribution tables
+    logging.info("Adding topical contributions")
+    for config in topical_contributions_config:
+        cats = config.get("categories", list(all_meetings.keys()))
+        meetings = sum((all_meetings[cat] for cat in cats[1:]), all_meetings[cats[0]])
+        meetings.add_topic(
+            target=config["target"],
+            include=config["include"],
+            exclude=config.get("exclude"),
+        )
+
+    # add minutes tables
+    for name, config in minutes_config.items():
+        logging.info(f"Adding minutes to {config['target']}")
+        all_meetings[name].minutes_md(name=name, target=config["target"])
+
+
+if __name__ == "__main__":
+    main()

indicomb2/markdown.py

@@ -0,0 +1,116 @@
+from pathlib import Path
+
+
+class Markdown:
+    """Class to write markdown for material for mkdocs."""
+
+    def __init__(self, md: str = "", hide_toc=False) -> None:
+        self.md = md
+        if hide_toc:
+            self.md += "\n".join(["---", "hide:", "  - toc", "---"])  # noqa: FLY002
+
+    def __add__(self, other):
+        if not other.startswith("\n") and self.md:
+            other = "\n" + other
+        self.md += other
+        return self
+
+    def __radd__(self, other):
+        if not self.md.startswith("\n\n") and other:
+            self.md = "\n\n" + self.md
+        return other + str(self)
+
+    def __str__(self) -> str:
+        return self.md
+
+    @staticmethod
+    def indent(text):
+        return "\t" + text.replace("\n", "\n\t")
+
+    @staticmethod
+    def header(text, level):
+        return "#" * level + " " + text + "\n"
+
+    @staticmethod
+    def paragraph(text):
+        return text + "\n\n"
+
+    @staticmethod
+    def code(text, language="", line_numbers=False):
+        ln = ""
+        if line_numbers:
+            ln += ' linenums="1"'
+        return "\n\n```" + language + ln + "\n" + text + "\n```\n\n"
+
+    @staticmethod
+    def link(text, url):
+        return "[" + text + "](" + url + ")"
+
+    @staticmethod
+    def image(text, url):
+        return "![" + text + "](" + url + ")"
+
+    @staticmethod
+    def table(data: dict[str, list], format_ints=True):
+        assert len({len(row) for row in data.values()}) == 1
+        text = "| "
+        for header in data:
+            text += header + " | "
+        text += "\n| "
+        for header in data:
+            text += "-" * len(header) + " | "
+        text += "\n"
+
+        for row_idx in range(len(next(iter(data.values())))):
+            text += "| "
+            for col_idx in range(len(data.keys())):
+                value = data[list(data.keys())[col_idx]][row_idx]
+                if (format_ints and (isinstance(value, str) and value.isdigit())) or isinstance(
+                    value, int
+                ):
+                    value = f"{int(value):,}"
+                text += str(value) + " | "
+            text += "\n"
+
+        text += "\n\n"
+        return text
+
+    @staticmethod
+    def list(items):
+        text = ""
+        for item in items:
+            text += "- " + item + "\n"
+        text += "\n\n"
+        return text
+
+    def admonition(self, style, header, body=None, expanded=False):
+        if body is None:
+            expanded = False
+        md = "\n"
+        md += "!!!" if body is None else "???"
+        if expanded:
+            md += "+"
+        md += " "
+        md += style + ' "' + header + '"\n\n'
+        if body:
+            md += self.indent(body) + "\n\n"
+        return md
+
+    def tabs(self, tabs: dict[str, str]):
+        text = "\n"
+        for header, body in tabs.items():
+            text += "=== " + f'"{header}"' + " \n\n"
+            text += self.indent(body) + "\n\n"
+        return text
+
+    @staticmethod
+    def bold_text(text):
+        return f"**{text}**"
+
+    @staticmethod
+    def italic_text(text):
+        return f"*{text}*"
+
+    def to_file(self, fname):
+        with Path(fname).open("w") as f:
+            f.write(self.md)

indicomb2/meetings.py

@@ -0,0 +1,91 @@
+import datetime
+from dataclasses import dataclass
+from pathlib import Path
+
+from indicomb2.event import Event
+from indicomb2.markdown import Markdown
+
+
+@dataclass
+class Meetings:
+    meetings: list[Event]
+    url: str
+    start_date: str
+
+    def __post_init__(self):
+        self.meetings = sorted(self.meetings)
+
+    def __add__(self, other):
+        return Meetings(self.meetings + other.meetings, self.url, self.start_date)
+
+    def meeting_list_md(self, name: str, when: str, target: Path | str):
+        target = Path(target)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        md = Markdown()
+        md += md.header(f"{name.capitalize()} Meetings", level=1)
+        name = md.link(name, self.url)
+        md += f"The {name} meetings take place every {when} CERN time."
+        md += "\n\nYou can find a meeting overview below.\n\n"
+        md += md.header("Meeting Overview", level=2)
+        last_updated = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
+        md += f"*last updated: {last_updated}*\n"
+        md += "\n\n".join([str(m) for m in self.meetings])
+        md += "\n\n"
+        md += (
+            f"*date cutoff: {self.start_date}, for older meetings please check "
+            f"[indico]({self.url})*\n"
+        )
+        with target.open("w") as f:
+            f.write(str(md))
+
+    def minutes_md(self, name: str, target: Path):
+        target = Path(target)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        md = Markdown()
+        md += md.header(f"{name.capitalize()} Meetings Minutes", level=1)
+        last_updated = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y-%m-%d %H:%M")
+        md += f"*last updated: {last_updated}*\n"
+        md += "\n\n".join([m.minutes() for m in self.meetings])
+        md += "\n\n"
+        md += (
+            f"*date cutoff: {self.start_date}, for older meetings please check "
+            f"[indico]({self.url})*\n"
+        )
+        md += md.link("indico", self.url)
+        with target.open("w") as f:
+            f.write(str(md))
+
+    def add_topic(
+        self,
+        target: Path,
+        include: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ):
+        if include is None:
+            include = []
+        if exclude is None:
+            exclude = []
+        data = {"Date": [], "Title": [], "Speakers": []}
+        for e in self.meetings:
+            for c in e.contributions:
+                if any(x.lower() in c.title.lower() for x in include) and not any(
+                    x.lower() in c.title.lower() for x in exclude
+                ):
+                    data["Date"].append(f"[{e.date}]({c.url})")
+                    data["Title"].append(c.title)
+                    data["Speakers"].append(c.speakers)
+
+        md = Markdown()
+        md += "\n"
+        md += md.header("Meeting Contributions", level=2)
+        md += md.table(data)
+        md += "\n\n"
+        md += f"*search terms: {include}*\n"
+        md += (
+            f"*date cutoff: {self.start_date}, for older meetings please check "
+            f"[indico]({self.url})*\n"
+        )
+        target = Path(target)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with target.open("w") as f:
+            f.write(str(md))

indicomb2-0.1.dist-info/METADATA

@@ -0,0 +1,40 @@
+Metadata-Version: 2.1
+Name: indicomb2
+Version: 0.1
+Summary: Faster indicomb with mkdocs markdown support
+License: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: pyyaml >=6.0.2
+Requires-Dist: requests >=2.32.3
+Provides-Extra: dev
+Requires-Dist: pytest >=8.3.3 ; extra == 'dev'
+Requires-Dist: ruff >=0.6.8 ; extra == 'dev'
+
+Faster indicomb with mkdocs markdown support.
+An alternative to the original [indicomb](https://gitlab.cern.ch/indicomb/indicomb).
+
+This is fast enough to run in the CI as your docs get built (say goodby to your cron jobs!), and comes with mkdocs markdown support.
+
+
+## Quick Start
+
+```bash
+pip install indicomb2
+```
+
+Set up a config, see the [example](https://gitlab.cern.ch/indicomb/indicomb2/-/blob/main/example.yaml)
+
+Run:
+
+```bash
+indicomb2 -c my_config.yaml
+```
+
+
+## More details
+
+1. Set up a CERN docs site: https://how-to.docs.cern.ch/
+2. Add an environment variable `INDICO_API_TOKEN` with the token from https://indico.cern.ch/user/tokens/
+3. Setup a config, see the [example](https://gitlab.cern.ch/indicomb/indicomb2/-/blob/main/example.yaml)
+4. Run `indicomb2 -c my_config.yaml`

indicomb2-0.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+indicomb2/__init__.py,sha256=Lgf9keBX7aG_JEFt6UYSyiFKcm-BYGtmk9ohpQhYpyk,20
+indicomb2/contribution.py,sha256=6V87pgNAObRq_C9pDv-ET_WKT5OmEQIo-9-jI7zCnuE,775
+indicomb2/event.py,sha256=rE8RBv64-cjshoC6ObJMRF94WY1y_4m1BRgK4K-0ZZo,1642
+indicomb2/indicomb2.py,sha256=ndG9dPY8W6l-HwfjNbHXOZHslYL1qZacBv5ELzKwfcQ,6193
+indicomb2/markdown.py,sha256=zSeqG7zpArFY69rLQ-OG7rleDgW59K37IS0aR6G-dI0,3179
+indicomb2/meetings.py,sha256=7Z3cJYwhNTYUG2dVkHEPKufTgSNp7C1BCrRBpv245jY,3281
+indicomb2-0.1.dist-info/METADATA,sha256=kqPJBsvaT3pkQhLqamWkVrWYxvGSQFEdKtN-UgS5fZM,1150
+indicomb2-0.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+indicomb2-0.1.dist-info/entry_points.txt,sha256=oPgYgF78V8fSyi_FM2MtKSfsV16iP1R6882Nzx5eRXw,55
+indicomb2-0.1.dist-info/top_level.txt,sha256=O5eDzpAdXFOhQsyuc7H7E7QfKzy82og64kSv_leiSBY,10
+indicomb2-0.1.dist-info/RECORD,,

indicomb2-0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

indicomb2-0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+indicomb2 = indicomb2.indicomb2:main

indicomb2-0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+indicomb2