     1	import inspect
     2	from functools import cache
     3	import shutil
     4	import time
     5	from dataclasses import asdict
     6	from functools import partial
     7	
     8	from engine import bus
     9	from engine.markers import plain
    10	from engine.record import Record
    11	from resources.base import PART_OF, PROJECT, SYSTEM, USER, Ref, Refused, Resource, SECTION, check_abstract, check_title
    12	from resources.shapes import Options, check, normalize_options, typed
    13	from controllers.discussion import TWICE_WITHIN, Discussed
    14	from controllers.files import Files
    15	from controllers.links import Links
    16	from controllers import marks
    17	from controllers.stored import RowStore
    18	from engine.wording import noun
    19	
    20	WORDS = ("title", "abstract", "brief")
    21	LAST = 25
    22	SEARCHABLE: dict[str, dict[int, tuple[float, str]]] = {}
    23	
    24	
    25	def searchable(r: Resource) -> str:
    26	    parts = [r.title, r.brief, r.abstract, *(f"{s[SECTION.title]} {s[SECTION.body]}" for s in r.sections),
    27	             *(f"{name} {tags}" for name, tags in r.files.items())]
    28	    return "\n".join(parts).lower()
    29	COMMANDS: dict[str, dict] = {}
    30	
    31	
    32	def networked(type_: str, name: str) -> bool:
    33	    held = COMMANDS.get(type_, {}).get(name) or getattr(CONTROLLERS.get(type_), name, None)
    34	    return bool(getattr(held, "network", False))
    35	HANDLERS: dict[str, list] = {}
    36	CONTROLLERS: dict[str, type] = {}
    37	
    38	
    39	def checked_field(fields: dict, key: str, value):
    40	    if key not in fields:
    41	        return value
    42	    return check(key, fields[key], normalize_options(value) if key == Options.options else value)
    43	
    44	@cache
    45	def actions(controller: type) -> tuple[str, ...]:
    46	    return tuple(sorted(name for name, f in inspect.getmembers(controller, inspect.isfunction) if getattr(f, "action", False)))
    47	
    48	
    49	def word_names(controller: type) -> set[str]:
    50	    return {*actions(controller), *COMMANDS.get(controller.resource.type, {})}
    51	
    52	
    53	def word_function(controller: type, name: str):
    54	    return COMMANDS.get(controller.resource.type, {}).get(name) or getattr(controller, name)
    55	
    56	
    57	def word_parameters(controller: type, name: str) -> list[inspect.Parameter]:
    58	    return list(inspect.signature(word_function(controller, name)).parameters.values())[1:]
    59	
    60	
    61	class Controller(Files, Links, Discussed):
    62	    resource = Resource
    63	    actor = "user"
    64	
    65	    def __init__(self, record: Record, actor: str | None = None, session: str = "", agent: str = "", force: str = ""):
    66	        self.record = record
    67	        self.rows = RowStore(record, self.resource, self._ordered, self._visible, self._also, self._damaged)
    68	        self.session = session
    69	        self.agent = agent
    70	        self.force = force
    71	        self.forced: list[str] = []
    72	        if actor:
    73	            self.actor = actor
    74	
    75	    def _refuse(self, why: str) -> None:
    76	        if not self.force:
    77	            raise Refused(why)
    78	        self.forced.append(why)
    79	
    80	    @property
    81	    def type(self) -> str:
    82	        return self.resource.type
    83	
    84	    def _note_force(self, r: Resource) -> None:
    85	        if not self.forced:
    86	            return
    87	        r.data["forced"] = [*(r.data.get("forced") or []),
    88	                            {"why": self.force, "past": list(self.forced), "who": self.actor, "at": time.time()}]
    89	        self.forced = []
    90	
    91	    def _guarded(self, r: Resource, action: str) -> None:
    92	        allowed = self.resource.editors.get(r.author)
    93	        if allowed is None or self.actor in allowed or not self.rows.exists(r.n):
    94	            return
    95	        stored = self.load(r.n)
    96	        parts = [section[SECTION.title] for section in stored.sections]
    97	        rewritten = any(getattr(stored, f) != getattr(r, f) for f in WORDS) or [section[SECTION.title] for section in r.sections][:len(parts)] != parts
    98	        if action == "deleted" or rewritten:
    99	            self._refuse(f"{self.type} {r.n} was written by the {stored.seen[0]}: answer it with journal {self.type} {self.resource.answer_command} {r.n} \"<text>\" instead of changing it")
   100	
   101	    def _shipped(self, r: Resource, action: str) -> None:
   102	        if self.actor == SYSTEM or not self.rows.exists(r.n):
   103	            return
   104	        stored = self.load(r.n)
   105	        if not stored.data.get("system"):
   106	            return
   107	        free = ("kept", *self.resource.progress)
   108	        kept = lambda row: {k: v for k, v in row.data.items() if k not in free}
   109	        if action in ("deleted", "completed") or any(getattr(stored, f) != getattr(r, f) for f in WORDS) or \
   110	                stored.sections != r.sections or kept(stored) != kept(r):
   111	            self._refuse(f"{self.type} {r.n} ships with the journal and cannot be changed or removed")
   112	
   113	    def save(self, r: Resource, action: str, **event) -> Resource:
   114	        self._shipped(r, action)
   115	        self._guarded(r, action)
   116	        r.rewrite(plain)
   117	        self._note_force(r)
   118	        if self.actor not in r.seen:
   119	            r.seen.append(self.actor)
   120	        r.updated = time.time()
   121	        self.rows.persist(r)
   122	        self._emit(r.n, action, **event)
   123	        return r
   124	
   125	    def _emit(self, n: int, action: str, **event):
   126	        return self.record.emit(self.type, n, action, self.actor, **event)
   127	
   128	    def _retitle(self, n: int, title: str) -> Resource:
   129	        return self.update(int(n), title=title.strip())
   130	
   131	    def _finished(self, r: Resource) -> bool:
   132	        return bool(r.completed)
   133	
   134	    def _unfinished(self, n: int, ended: str) -> Resource:
   135	        row = self.load(n)
   136	        if row.completed:
   137	            raise Refused(f"{self.type} {n} is {ended}")
   138	        return row
   139	
   140	    def _handled(self, action: str, /, **args):
   141	        for fn in HANDLERS.get(f"{self.type}.{action}", []) + HANDLERS.get(action, []):
   142	            taken = fn(self, **args)
   143	            if taken is not None:
   144	                return taken
   145	        return None
   146	
   147	    def _field_choices(self, r: Resource) -> dict:
   148	        return {}
   149	
   150	    def _shaped(self, data: dict) -> dict:
   151	        fields = self.resource.fields
   152	        return {k: checked_field(fields, k, v) for k, v in data.items()}
   153	
   154	    def _given(self, data: dict) -> Resource:
   155	        return self.resource(data=self._shaped(data))
   156	
   157	    def _twin(self, title: str, brief: str, about, idempotency: str = "") -> Resource | None:
   158	        if not self.resource.deduplicates:
   159	            return None
   160	        since = time.time() - TWICE_WITHIN
   161	        lately = [row["n"] for row in self.rows.summaries() if not row["deleted"] and row["updated"] >= since]
   162	        recent = (self.load(n) for n in reversed(lately[-20:]))
   163	        return next((r for r in recent if r.created >= since and r.title == title and r.brief == brief
   164	                     and r.author == self.actor and (not about or about in r.refs)
   165	                     and (not idempotency or r.data.get("idempotency") == idempotency)), None)
   166	
   167	    @marks.action
   168	    def create(self, title: str, abstract: str = "", brief: str = "", **data) -> Resource:
   169	        twin = self._twin(title, brief, data.get("about"), data.get("idempotency", ""))
   170	        if twin is not None:
   171	            return twin
   172	        taken = self._handled("create", title=title, abstract=abstract, brief=brief, **data)
   173	        if taken is not None:
   174	            return taken
   175	        for name in self.resource.required:
   176	            if not data.get(name):
   177	                self._refuse(f"a {self.type} needs {name}: --set {name}=\"<word>,<word>\"")
   178	        with self.record.locked(self.resource.scope):
   179	            n = (self.rows.numbers() or [0])[-1] + 1
   180	            about, supersedes = data.pop("about", None), data.pop("supersedes", 0)
   181	            fields = self._shaped(data)
   182	            if self.agent:
   183	                fields.update(agent=self.agent, dispatcher=self.session)
   184	            if self.resource.scope == PROJECT:
   185	                fields["environment"] = self.record.env
   186	            r = self.resource(n=n, title=check_title(title), abstract=check_abstract(abstract), brief=brief,
   187	                              data=fields, created=time.time(), seen=[self.actor], refs=[about] if about else [])
   188	            r = self.save(r, "created")
   189	            return self._supersede(int(supersedes), n) if supersedes else r
   190	
   191	    def _supersede(self, old: int, new: int) -> Resource:
   192	        self.complete(old, how=f"superseded by {self.type} {new}")
   193	        return self.link(new, f"{self.type}:{old}")
   194	
   195	    def _created_with_sections(self, title: str, abstract: str, brief: str, sections: list[tuple[str, str]], **data) -> Resource:
   196	        made = self.create(title, abstract, brief, **data)
   197	        for heading, body in sections:
   198	            made = self.section(made.n, heading, body)
   199	        return made
   200	
   201	    def _carried(self, n: int, into: "Controller", how: str) -> Resource:
   202	        r = self.load(n)
   203	        made = into._created_with_sections(r.title, r.abstract, r.brief, [(s[SECTION.title], s[SECTION.body]) for s in r.sections], **r.data)
   204	        self.complete(n, how=f"{how} {into.type} {made.n}")
   205	        return made
   206	
   207	    def _damaged(self, path: str, error: str) -> None:
   208	        from controllers.faults import damaged
   209	        damaged(self.record, path, error)
   210	
   211	    def _stopping(self, n: int, **asked) -> Resource:
   212	        return self.update(n, stopping={**asked, "at": time.time()})
   213	
   214	    @marks.action
   215	    def update(self, n: int, title: str | None = None, abstract: str | None = None, brief: str | None = None, outcome: str | None = None, **data) -> Resource:
   216	        taken = self._handled("update", n=n, title=title, abstract=abstract, brief=brief, outcome=outcome, **data)
   217	        if taken is not None:
   218	            return taken
   219	        with self.record.locked(self.resource.scope):
   220	            r = self.load(n)
   221	            if title is not None:
   222	                r.title = check_title(title)
   223	            if abstract is not None:
   224	                r.abstract = check_abstract(abstract)
   225	            if brief is not None:
   226	                r.brief = brief
   227	            if outcome is not None:
   228	                r.outcome = outcome
   229	            r.data.update(self._shaped(data))
   230	            given = {"title": title, "abstract": abstract, "brief": brief, "outcome": outcome}
   231	            return self.save(r, "updated", fields=[*(k for k, v in given.items() if v is not None), *data])
   232	
   233	    @marks.action
   234	    def stamp(self, n: int, **data) -> Resource:
   235	        return self._changed(n, "stamped", data, quiet=True, fields=sorted(data))
   236	
   237	    def appended(self, r: Resource, field: str, entry, keep: int, **data) -> Resource:
   238	        return self.update(r.n, **{field: [*(r.data.get(field) or []), entry][-keep:]}, **data)
   239	
   240	    def _changed(self, n: int, action: str, data: dict, **event) -> Resource:
   241	        with self.record.locked(self.resource.scope):
   242	            r = self.load(n)
   243	            r.data.update(self._shaped(data))
   244	            return self.save(r, action, **event)
   245	
   246	    @marks.action
   247	    def set(self, n: int, key: str, value: str) -> Resource:
   248	        return self.update(n, **{key: typed(value)})
   249	
   250	    def add_part(self, n: int, title: str, body: str) -> Resource:
   251	        return self.section(n, title, body)
   252	
   253	    @marks.action
   254	    def section(self, n: int, title: str, body: str) -> Resource:
   255	        with self.record.locked(self.resource.scope):
   256	            r = self.load(n)
   257	            for s in r.sections:
   258	                if s[SECTION.title] == title:
   259	                    s[SECTION.body] = body
   260	                    break
   261	            else:
   262	                r.sections.append({SECTION.title: title, SECTION.body: body})
   263	            return self.save(r, "updated", section=title)
   264	
   265	    @marks.action
   266	    def delete(self, n: int, why: str = "") -> Resource:
   267	        taken = self._handled("delete", n=n, why=why)
   268	        if taken is not None:
   269	            return taken
   270	        with self.record.locked(self.resource.scope):
   271	            r = self.load(n)
   272	            r.deleted = time.time()
   273	            return self.save(r, "deleted", why=why)
   274	
   275	    @marks.action
   276	    def complete(self, n: int, how: str = "", **data) -> Resource:
   277	        taken = self._handled("complete", n=n, how=how, **data)
   278	        if taken is not None:
   279	            return taken
   280	        with self.record.locked(self.resource.scope):
   281	            r = self.load(n)
   282	            if r.completed:
   283	                self._refuse(f"{self.type} {n} is already closed")
   284	            r.completed = time.time()
   285	            r.outcome = how
   286	            r.data.update(self._shaped(data))
   287	            if self.resource.lists_completed_unread and self.actor != USER:
   288	                r.seen = [who for who in r.seen if who != USER]
   289	            return self.save(r, "completed", how=how, **data)
   290	
   291	    @marks.action
   292	    def reopen(self, n: int, why: str) -> Resource:
   293	        with self.record.locked(self.resource.scope):
   294	            r = self.load(n)
   295	            if r.deleted:
   296	                self._refuse(f"{self.type} {n} is archived; restore it before reopening it")
   297	            if not r.completed:
   298	                self._refuse(f"{self.type} {n} is not {self.named('complete')}")
   299	            r.completed = 0.0
   300	            r.outcome = ""
   301	            return self.save(r, "reopened", why=why)
   302	
   303	    def named(self, method: str) -> str:
   304	        return self.resource.command_names.get(method, method)
   305	
   306	    def method(self, name: str):
   307	        if name in self.resource.command_names and name not in self.resource.command_names.values():
   308	            raise Refused(f"a {self.type} calls that {self.resource.command_names[name]}")
   309	        return self.action(name)
   310	
   311	    def action(self, name: str):
   312	        if name.startswith("_") or name not in {*word_names(type(self)), *self.resource.command_names.values()}:
   313	            raise Refused(f"{self.type} has no action {name!r}")
   314	        command = COMMANDS.get(self.type, {}).get(name)
   315	        if command:
   316	            return bus.commanded(self.type, name, partial(command, self))
   317	        method = next((method for method, alias in self.resource.command_names.items() if alias == name), name)
   318	        return bus.commanded(self.type, method, getattr(self, method))
   319	
   320	    @marks.action
   321	    def restore(self, n: int) -> Resource:
   322	        r = self.load(n)
   323	        r.deleted = 0.0
   324	        return self.save(r, "updated", restored=True)
   325	
   326	    def _prune(self) -> None:
   327	        prunable = sorted((r for r in self.rows.every(deleted=True) if self.resource.pruned_when.admits(r)), key=lambda r: r.created, reverse=True)
   328	        for r in prunable[self.resource.kept:]:
   329	            self.force_delete(r.n)
   330	
   331	    @marks.action
   332	    def force_delete(self, n: int) -> None:
   333	        self.load(n)
   334	        self.rows.remove(n)
   335	        files = self.rows.row_folder(n)
   336	        if files.is_dir():
   337	            shutil.rmtree(files)
   338	        self._emit(n, "deleted", force=True)
   339	
   340	    @marks.action
   341	    def move(self, n: int, env: str) -> Resource:
   342	        r = self.load(n)
   343	        self._shipped(r, "deleted")
   344	        self._guarded(r, "deleted")
   345	        there = type(self)(Record(self.record.root, env), actor=self.actor)
   346	        with there.record.locked(self.resource.scope):
   347	            m = (there.rows.numbers() or [0])[-1] + 1
   348	            moved = self.resource(**{**asdict(r), "n": m})
   349	            if any(self.folder(n).iterdir()):
   350	                shutil.copytree(self.folder(n), there.folder(m), dirs_exist_ok=True)
   351	            there.save(moved, "created", moved_from=f"{self.record.env}/{n}")
   352	        self.delete(n, why=f"moved to {env} as {self.type} {m}")
   353	        return moved
   354	
   355	    @marks.action
   356	    def show(self, n: int) -> Resource:
   357	        row = self.read(n)
   358	        self._handled("show", row=row)
   359	        return row
   360	
   361	    @marks.action
   362	    def read(self, n: int) -> Resource:
   363	        return self.read_all([n])[0]
   364	
   365	    @marks.action
   366	    def read_all(self, numbers: list[int]) -> list[Resource]:
   367	        rows = []
   368	        changed = []
   369	        wanted = list(dict.fromkeys(int(n) for n in numbers))
   370	        with self.record.locked(self.resource.scope):
   371	            for n in wanted:
   372	                r = self.load(n)
   373	                rows.append(r)
   374	                if self.actor in r.seen:
   375	                    continue
   376	                r.seen.append(self.actor)
   377	                r.updated = time.time()
   378	                self.rows.persist(r)
   379	                changed.append(r)
   380	            if changed:
   381	                self._emit(changed[0].n, "updated", numbers=[r.n for r in changed], seen=self.actor, by="read")
   382	        return rows
   383	
   384	    @marks.action
   385	    def unread(self, actor: str | None = None) -> list[Resource]:
   386	        who = actor or self.actor
   387	        return [self.load(row["n"]) for row in self.rows.summaries() if who not in row["seen"] and not row["completed"] and not row["deleted"]]
   388	
   389	    @marks.action
   390	    def all(self, deleted: bool = False, completed: bool = False, last: int = LAST) -> list[Resource]:
   391	        if not deleted and int(last) and type(self)._ordered is Controller._ordered:
   392	            listed = [row["n"] for row in self.rows.summaries() if not row["deleted"] and (not row["completed"] or completed and not row.get(PART_OF))]
   393	            return [self.rows.peek(n) for n in listed[-int(last):]]
   394	        rows = self.rows.every(deleted) if completed or deleted else self.rows.standing()
   395	        rows = rows if completed else [r for r in rows if not r.completed]
   396	        return rows[-int(last):] if int(last) else rows
   397	
   398	    def mark(self, r: Resource) -> str:
   399	        return ""
   400	
   401	    @marks.action
   402	    def search(self, term: str) -> list[Resource]:
   403	        want = term.lower()
   404	        texts = self._texts()
   405	        hits = (row["n"] for row in reversed(self.rows.summaries()) if not row["deleted"] and want in texts.get(row["n"], ""))
   406	        return [self.rows.peek(n) for n, _ in zip(hits, range(LAST))]
   407	
   408	    def load(self, n: int | str) -> Resource:
   409	        return self.rows.load(n)
   410	
   411	    def path(self, n: int):
   412	        return self.rows.path(n)
   413	
   414	    def _ordered(self, rows: list[Resource]) -> list[Resource]:
   415	        return rows
   416	
   417	    def _visible(self, row: Resource) -> bool:
   418	        return True
   419	
   420	    def _also(self) -> list[Resource]:
   421	        return []
   422	
   423	    def _warm(self) -> None:
   424	        self.rows.warm()
   425	        self._texts()
   426	
   427	    def _texts(self) -> dict[int, str]:
   428	        kept = SEARCHABLE.setdefault(str(self.rows.folder()), {})
   429	        for row in self.rows.summaries():
   430	            if row["deleted"] or kept.get(row["n"], (None,))[0] == row["updated"]:
   431	                continue
   432	            kept[row["n"]] = (row["updated"], searchable(self.rows.peek(row["n"])))
   433	        return {n: text for n, (_, text) in kept.items()}
   434	
   435	    @marks.action
   436	    def find(self, name: str) -> Resource:
   437	        if str(name).isdigit():
   438	            return self.load(name)
   439	        hits = [row["n"] for row in self.rows.summaries() if not row["deleted"] and name.lower() in row["title"].lower()]
   440	        if len(hits) != 1:
   441	            raise Refused(f"{'no' if not hits else len(hits)} {noun(len(hits), self.type)} match {name!r}" + ("; say more of the title" if len(hits) > 1 else ""))
   442	        return self.load(hits[0])
   443	
   444	
   445	def controller_of(record: Record, ref: "str | Ref", actor: str = SYSTEM) -> "Controller":
   446	    ref = Ref.parse(ref)
   447	    if ref.type not in CONTROLLERS:
   448	        raise Refused(f"{str(ref)!r} is not a row: write it as type:number, like todo:785")
   449	    return CONTROLLERS[ref.type](record, actor=actor)
   450	
   451	
   452	def row_of(record: Record, ref: "str | Ref") -> Resource:
   453	    return controller_of(record, ref).load(Ref.parse(ref).n)
   454	
   455	
   456	def register(*classes) -> None:
   457	    CONTROLLERS.update({c.resource.type: c for c in classes})
     1	import os
     2	import time
     3	import zipfile
     4	from bisect import bisect_left
     5	from dataclasses import dataclass
     6	from pathlib import Path
     7	from resources.base import LAZY, MEMORY, OWNER, PART_OF, Missing, Refused, Resource
     8	from engine.stored import append_text, read_json, write_json, write_text
     9	from engine.memo import Memo
    10	
    11	DAMAGED = "damaged"
    12	DRAFT_OF = "draft_of"
    13	
    14	INDEX = "index.json"
    15	CHANGES = "changes.log"
    16	PACKED = "packed"
    17	ARCHIVE = "zip"
    18	
    19	
    20	def wholes(rows: list, part_of) -> list:
    21	    return [row for row in rows if not part_of(row)]
    22	SUMMARIES: dict[str, tuple] = {}
    23	HELD = Memo()
    24	PACKS = Memo()
    25	INDEXED: dict[str, dict] = {}
    26	STAMPED: dict[str, "Stamped"] = {}
    27	STAMPS_FRESH = 60.0
    28	WRITTEN: dict[str, float] = {}
    29	FLUSH_ROWS, FLUSH_SECONDS = 200, 300.0
    30	OPEN: dict[str, tuple] = {}
    31	
    32	
    33	def mtime(path: Path) -> int:
    34	    try:
    35	        return path.stat().st_mtime_ns
    36	    except OSError:
    37	        return 0
    38	
    39	
    40	def numbered(n: int) -> str:
    41	    return f"{n:03d}"
    42	
    43	
    44	def member(n: int) -> str:
    45	    return f"{numbered(n)}.md"
    46	
    47	
    48	def stamp_of(found: os.stat_result) -> str:
    49	    return f"{found.st_mtime_ns}-{found.st_size}"
    50	
    51	
    52	def is_part(row: dict) -> bool:
    53	    return bool(row.get(PART_OF) or row.get(DRAFT_OF))
    54	
    55	
    56	def opened(archive: Path) -> zipfile.ZipFile:
    57	    stamp = mtime(archive)
    58	    held = OPEN.get(str(archive))
    59	    if not held or held[0] != stamp:
    60	        if held:
    61	            held[1].close()
    62	        held = OPEN[str(archive)] = (stamp, zipfile.ZipFile(archive))
    63	    return held[1]
    64	
    65	
    66	
    67	@dataclass(frozen=True)
    68	class Stamped:
    69	    mark: int
    70	    checked: float
    71	    stamps: dict
    72	    inodes: dict
    73	    noted: int
    74	
    75	class RowStore:
    76	    def __init__(self, record, resource, order, visible, also, on_damage):
    77	        self.record = record
    78	        self.resource = resource
    79	        self.type = resource.type
    80	        self.order = order
    81	        self.visible = visible
    82	        self.also = also
    83	        self.on_damage = on_damage
    84	
    85	    def path(self, n: int) -> Path:
    86	        return self.row_folder(n) / f"{self.type}.md" if self.resource.own_folder else self.folder() / member(n)
    87	
    88	    def folder(self) -> Path:
    89	        return self.record.folder(self.type, self.resource.scope)
    90	
    91	    def row_folder(self, n: int) -> Path:
    92	        return self.folder() / numbered(n)
    93	
    94	    def write_file(self, r: Resource) -> None:
    95	        p = self.path(r.n)
    96	        if self.resource.own_folder:
    97	            p.parent.mkdir(parents=True, exist_ok=True)
    98	        else:
    99	            self._note(r.n)
   100	        write_text(p, r.dump())
   101	        if self.resource.own_folder:
   102	            os.utime(self.folder())
   103	
   104	    def persist(self, r: Resource) -> None:
   105	        folder = self.folder()
   106	        before = self._moved(folder) if folder.is_dir() else None
   107	        self.write_file(r)
   108	        self.reindexed(r.n, before, r)
   109	
   110	    def _note(self, n: int) -> None:
   111	        append_text(self.folder() / CHANGES, f"{n}\n")
   112	
   113	    @staticmethod
   114	    def _noted_end(folder: Path) -> int:
   115	        try:
   116	            return (folder / CHANGES).stat().st_size
   117	        except OSError:
   118	            return 0
   119	
   120	    def _noted(self, folder: Path, since: int) -> tuple[set[int], int]:
   121	        try:
   122	            with (folder / CHANGES).open("rb") as changes:
   123	                changes.seek(0, os.SEEK_END)
   124	                end = changes.tell()
   125	                if end <= since:
   126	                    return set(), end
   127	                changes.seek(since)
   128	                read = changes.read(end - since).decode()
   129	        except OSError:
   130	            return set(), 0
   131	        whole = read[:read.rfind("\n") + 1]
   132	        return {int(line) for line in whole.splitlines() if line.isdigit()}, since + len(whole.encode())
   133	
   134	    def numbers(self) -> list[int]:
   135	        folder = self.folder()
   136	        return sorted(set(self._stamps(folder)) | set(self.packed()))
   137	
   138	    def summaries(self) -> list[dict]:
   139	        folder = self.folder()
   140	        moved = self._moved(folder)
   141	        held = SUMMARIES.get(str(folder))
   142	        if held and held[0] == moved:
   143	            return held[1]
   144	        loose = self._loose(folder)
   145	        if held and held[0][1] == moved[1]:
   146	            touched = self._differing(held[1], loose)
   147	            if len(touched) < FLUSH_ROWS:
   148	                return self._patched(folder, moved, held[1], loose, touched)
   149	        return self._summarised(folder, moved, [loose[n] for n in sorted(loose) if not loose[n].get(DAMAGED)])
   150	
   151	    def _differing(self, held: list[dict], loose: dict[int, dict]) -> set[int]:
   152	        listed = {row["n"]: row for row in held}
   153	        packed = self.packed()
   154	        shown = {n: row for n, row in loose.items() if not (row.get(DAMAGED) or is_part(row))}
   155	        return {n for n, row in shown.items() if listed.get(n) is not row} | {n for n in listed if n not in shown and n not in packed}
   156	
   157	    def _patched(self, folder: Path, moved: tuple, held: list[dict], loose: dict[int, dict], touched: set[int]) -> list[dict]:
   158	        rows = list(held)
   159	        for n in sorted(touched):
   160	            at = bisect_left(rows, n, key=lambda row: row["n"])
   161	            if at < len(rows) and rows[at]["n"] == n:
   162	                del rows[at]
   163	            row = loose.get(n) or self.packed().get(n)
   164	            if row and not row.get(DAMAGED) and not is_part(row):
   165	                rows.insert(at, row)
   166	        SUMMARIES[str(folder)] = (moved, rows)
   167	        return rows
   168	
   169	    def _moved(self, folder: Path) -> tuple:
   170	        return (folder.stat().st_mtime_ns, mtime(folder / PACKED / INDEX))
   171	
   172	    def _summarised(self, folder: Path, moved: tuple, loose: list[dict]) -> list[dict]:
   173	        seen = {row["n"] for row in loose}
   174	        packed = [row for n, row in self.packed().items() if n not in seen]
   175	        rows = wholes(sorted(loose + packed, key=lambda row: row["n"]), is_part)
   176	        SUMMARIES[str(folder)] = (moved, rows)
   177	        return rows
   178	
   179	    def reindexed(self, n: int, before: tuple | None, r: Resource | None = None) -> None:
   180	        folder = self.folder()
   181	        held, known = SUMMARIES.get(str(folder)), INDEXED.get(str(folder))
   182	        if not held or known is None or held[0] != before:
   183	            return
   184	        rows = list(held[1])
   185	        at = bisect_left(rows, n, key=lambda row: row["n"])
   186	        if at < len(rows) and rows[at]["n"] == n:
   187	            del rows[at]
   188	        if r is None:
   189	            known.pop(n, None)
   190	        else:
   191	            known[n] = self._row(r, stamp_of(self.path(n).stat()))
   192	            if not is_part(known[n]):
   193	                rows.insert(at, known[n])
   194	        SUMMARIES[str(folder)] = (self._moved(folder), rows)
   195	
   196	    def _row(self, r: Resource, stamp: str) -> dict:
   197	        return {"n": r.n, "title": r.title, "deleted": r.deleted, "completed": r.completed, "seen": r.seen, "refs": r.refs, "updated": r.updated,
   198	                "files": len(r.files), PART_OF: r.data.get(PART_OF, ""), DRAFT_OF: r.data.get(DRAFT_OF, ""), OWNER: r.data.get(OWNER, ""),
   199	                **{k: r.data.get(k) for k in self.resource.indexed}, "stamp": stamp}
   200	
   201	    def packed(self) -> dict[int, dict]:
   202	        index = self.folder() / PACKED / INDEX
   203	        stamp = mtime(index)
   204	        if not stamp:
   205	            return {}
   206	        return PACKS.get(str(index), stamp, lambda: {int(n): row for n, row in read_json(index, dict, {}).items()})
   207	
   208	    def _stamps(self, folder: Path) -> dict[int, str]:
   209	        if not self.resource.own_folder:
   210	            mark, now = os.stat(folder).st_mtime_ns, time.monotonic()
   211	            held = STAMPED.get(str(folder))
   212	            fresh = held and now - held.checked < STAMPS_FRESH
   213	            if fresh and held.mark == mark:
   214	                return held.stamps
   215	            if fresh:
   216	                changed, noted = self._noted(folder, held.noted)
   217	                if changed:
   218	                    return self._restamped(folder, mark, held, changed, noted)
   219	            noted = self._noted_end(folder)
   220	            stamps, inodes = {}, {}
   221	            for e in os.scandir(folder):
   222	                if not (e.name.endswith(".md") and e.name[:-3].isdigit()):
   223	                    continue
   224	                n = int(e.name[:-3])
   225	                inodes[n] = e.inode()
   226	                if fresh and held.inodes.get(n) == inodes[n]:
   227	                    stamps[n] = held.stamps[n]
   228	                else:
   229	                    stamps[n] = stamp_of(e.stat())
   230	            STAMPED[str(folder)] = Stamped(mark, held.checked if fresh else now, stamps, inodes, noted)
   231	            return stamps
   232	        stamps = {}
   233	        for e in os.scandir(folder):
   234	            if not e.is_dir() or not e.name.isdigit():
   235	                continue
   236	            try:
   237	                found = os.stat(os.path.join(e.path, f"{self.type}.md"))
   238	            except OSError:
   239	                continue
   240	            stamps[int(e.name)] = stamp_of(found)
   241	        return stamps
   242	
   243	    def _restamped(self, folder: Path, mark: int, held: Stamped, changed: set[int], noted: int) -> dict[int, str]:
   244	        stamps, inodes = dict(held.stamps), dict(held.inodes)
   245	        for n in changed:
   246	            try:
   247	                found = self.path(n).stat()
   248	            except OSError:
   249	                stamps.pop(n, None)
   250	                inodes.pop(n, None)
   251	                continue
   252	            stamps[n], inodes[n] = stamp_of(found), found.st_ino
   253	        STAMPED[str(folder)] = Stamped(mark, held.checked, stamps, inodes, noted)
   254	        return stamps
   255	
   256	    def _indexed(self, folder: Path) -> list[dict]:
   257	        rows = self._loose(folder)
   258	        return [rows[n] for n in sorted(rows) if not rows[n].get(DAMAGED)]
   259	
   260	    def _loose(self, folder: Path) -> dict[int, dict]:
   261	        stamps = self._stamps(folder)
   262	        known = INDEXED.get(str(folder)) or {int(n): row for n, row in read_json(folder / INDEX, dict, {}).items()}
   263	        needed = {"files", PART_OF, DRAFT_OF, OWNER, *self.resource.indexed}
   264	        rows = {}
   265	        for n, stamp in stamps.items():
   266	            row = known.get(n)
   267	            if row is not None and row.get("stamp") == stamp and (DAMAGED in row or needed <= row.keys()):
   268	                rows[n] = row
   269	                continue
   270	            try:
   271	                r = self.load(n)
   272	            except (Refused, OSError) as error:
   273	                rows[n] = {"n": n, DAMAGED: True, "stamp": stamp}
   274	                self.on_damage(str(self.path(n)), str(error))
   275	                continue
   276	            rows[n] = self._row(r, stamp)
   277	        changed = sum(1 for n, row in rows.items() if known.get(n) is not row) + len(known.keys() - rows.keys())
   278	        due = changed >= FLUSH_ROWS or time.time() - WRITTEN.get(str(folder), 0.0) >= FLUSH_SECONDS or not (folder / INDEX).is_file()
   279	        if changed and due:
   280	            write_json(folder / INDEX, rows)
   281	            WRITTEN[str(folder)] = time.time()
   282	        INDEXED[str(folder)] = rows
   283	        return rows
   284	
   285	    def by_title(self, title: str, standing: bool = False) -> Resource | None:
   286	        found = next((row["n"] for row in self.summaries() if row["title"] == title and not row["deleted"] and not (standing and row["completed"])), None)
   287	        return self.load(found) if found else None
   288	
   289	    def load(self, n: int | str) -> Resource:
   290	        r = self.peek(int(n))
   291	        return r.fork() if self.resource.loading == MEMORY else r
   292	
   293	    def peek(self, n: int) -> Resource:
   294	        p = self.path(n)
   295	        try:
   296	            found = p.stat()
   297	            stamp, where = (found.st_mtime_ns, found.st_size), str(p)
   298	        except OSError as error:
   299	            entry = self.packed().get(n)
   300	            if not entry:
   301	                raise Missing(f"no {self.type} {n}") from error
   302	            archive = self.folder() / PACKED / entry[ARCHIVE]
   303	            stamp, where = (mtime(archive), 0), f"{archive}:{n}"
   304	        if self.resource.loading != MEMORY:
   305	            return self._parsed(n)
   306	        return HELD.get(where, stamp, lambda: self._parsed(n))
   307	
   308	    def text(self, n: int) -> str:
   309	        p = self.path(n)
   310	        if p.is_file():
   311	            return p.read_text()
   312	        entry = self.packed().get(n)
   313	        if not entry:
   314	            raise Refused(f"no {self.type} {n}")
   315	        archive = self.folder() / PACKED / entry[ARCHIVE]
   316	        try:
   317	            return opened(archive).read(member(n)).decode()
   318	        except (OSError, KeyError, zipfile.BadZipFile) as error:
   319	            raise Refused(f"{self.type} {n} is missing from {archive}") from error
   320	
   321	    def _parsed(self, n: int) -> Resource:
   322	        try:
   323	            return self.resource.load(self.text(n))
   324	        except (ValueError, TypeError) as error:
   325	            raise Refused(f"{self.type} {n} is damaged: {self.path(n)}") from error
   326	
   327	    def exists(self, n: int) -> bool:
   328	        return self.path(n).is_file() or n in self.packed()
   329	
   330	    def remove(self, n: int) -> None:
   331	        folder = self.folder()
   332	        before = self._moved(folder) if folder.is_dir() else None
   333	        HELD.forget(str(self.path(n)))
   334	        self.path(n).unlink(missing_ok=True)
   335	        if self.resource.own_folder and folder.is_dir():
   336	            os.utime(folder)
   337	        elif folder.is_dir():
   338	            self._note(n)
   339	        packed = self.packed()
   340	        if n in packed:
   341	            write_json(folder / PACKED / INDEX, {k: row for k, row in packed.items() if k != n})
   342	        self.reindexed(n, before)
   343	
   344	    def pack(self, before: float) -> int:
   345	        folder = self.folder()
   346	        chosen = [row for row in self._indexed(folder) if (row["completed"] or row["deleted"]) and row["updated"] < before]
   347	        days: dict[str, list[dict]] = {}
   348	        for row in chosen:
   349	            days.setdefault(time.strftime("%Y-%m-%d", time.localtime(row["updated"])), []).append(row)
   350	        with self.record.locked(self.resource.scope):
   351	            for day, rows in days.items():
   352	                self._packed_into(folder, f"{day}.zip", rows)
   353	        return len(chosen)
   354	
   355	    def _packed_into(self, folder: Path, name: str, rows: list[dict]) -> None:
   356	        archive = folder / PACKED / name
   357	        archive.parent.mkdir(parents=True, exist_ok=True)
   358	        texts = {row["n"]: (folder / member(row["n"])).read_bytes() for row in rows}
   359	        index = self.packed()
   360	        kept = {n for n, entry in index.items() if entry[ARCHIVE] == name and n not in texts}
   361	        building = archive.with_suffix(".new")
   362	        with zipfile.ZipFile(building, "w", zipfile.ZIP_DEFLATED) as out:
   363	            for n in sorted(kept):
   364	                out.writestr(member(n), opened(archive).read(member(n)))
   365	            for n, text in texts.items():
   366	                out.writestr(member(n), text)
   367	        with zipfile.ZipFile(building) as check:
   368	            if any(check.read(member(n)) != text for n, text in texts.items()):
   369	                building.unlink()
   370	                raise OSError(f"{building} did not read back as written")
   371	        building.replace(archive)
   372	        write_json(folder / PACKED / INDEX, {**index, **{row["n"]: {**{k: v for k, v in row.items() if k != "stamp"}, ARCHIVE: name} for row in rows}})
   373	        for row in rows:
   374	            p = folder / member(row["n"])
   375	            if p.is_file() and p.read_bytes() == texts[row["n"]]:
   376	                HELD.forget(str(p))
   377	                p.unlink()
   378	
   379	    def warm(self) -> None:
   380	        rows = self.summaries()
   381	        if self.resource.loading == LAZY:
   382	            return
   383	        for row in rows:
   384	            if self.resource.loading == MEMORY:
   385	                self.load(row["n"])
   386	
   387	    def viewed(self) -> list[Resource]:
   388	        return [self.peek(row["n"]) for row in self.summaries() if not row["deleted"]]
   389	
   390	    def every(self, deleted: bool = False) -> list[Resource]:
   391	        memo = self.record.memo
   392	        if memo is None or (self.type, deleted) not in memo:
   393	            rows = [self.peek(row["n"]) for row in self.summaries()]
   394	            rows = wholes([r for r in rows if deleted or not r.deleted], lambda r: r.data.get(PART_OF))
   395	            if memo is None:
   396	                return self.order(rows)
   397	            memo[self.type, deleted] = rows
   398	        return self.order(list(memo[self.type, deleted]))
   399	
   400	    def standing(self, closed_since: float = 0, closed_last: int = 0) -> list[Resource]:
   401	        own = [r for r in self.kept(closed_since, closed_last) if (self.resource.hidden_listed or not r.hidden) and self.visible(r)]
   402	        return [*own, *self.also()]
   403	
   404	    def kept(self, closed_since: float = 0, closed_last: int = 0) -> list[Resource]:
   405	        rows = [row for row in self.summaries() if not row["deleted"]]
   406	        closed = [row for row in rows if row["completed"] and closed_since and row["completed"] >= closed_since]
   407	        kept = sorted(closed, key=lambda row: row["completed"])[-closed_last:] if closed_last else closed
   408	        return self.order([self.peek(row["n"]) for row in rows if not row["completed"]] + [self.peek(row["n"]) for row in kept])
