1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
#!/usr/bin/env python3
"""
Extract the reference link definitions, and uses, from a .md file.
They are extracted *without normalisation* - in particular,
without case folding. This is contrary to markdown semantics,
but it is desirable if we want to retain the original case.
When run as a program, prints a json document
{
"used": ["anchor", ...],
"defined"`: {"anchor": ["target", "title"] }
}
("title" can be null instead)
"""
# Basically all markdown parsers seem to treat undefined [foo]
# link references as literal text, including the [ ].
# I investigated several parsers including pandoc, marked (JS),
# and python3-markdown, and none of them seemed to have a way to
# override this or extract a list of apparently-unreferenced links.
#
# mistune has a hook mechanism, which we can abuse to insert
# instrumentation that spots when link definitions are queried,
# during processing.
import mistune # type: ignore
from typing import Tuple
class Tracking:
"""
Data structure which tracks used and defined keys.
You may access the properties `used` and `defined`;
`defined` mas each key to `(target, title)`.
`used` is a map from keys to `True`,
The keys here are *un*normalised, so they have not been lowercased.
"""
defined: dict[str, Tuple[str, str]] = {}
used: dict[str, bool] = {}
def as_json(self):
return json.dumps(
{
"used": list(self.used.keys()),
"defined": self.defined,
}
)
class TrackingBlockParser(mistune.BlockParser):
def __init__(self, track):
self.track = track
super().__init__()
def parse_def_link(self, m, state):
k = m.group(1)
t = m.group(2)
title = m.group(3)
self.track.defined[k] = (t, title)
return super().parse_def_link(m, state)
class TrackingInlineParser(mistune.InlineParser):
def __init__(self, track):
self.track = track
super().__init__()
def parse_ref_link(self, m, state):
k = m.group(2) or m.group(1)
self.track.used[k] = True
return super().parse_ref_link(m, state)
def extract_links(md_string):
"""
Given a markdown file, as a string, returns a `TrackingDict`
containing information about its ref links.
"""
track = Tracking()
# Our construction is reaching into the mistune innards more than ideal.
# It works with Debian's python3-mistune 3.1.3-1.
md = mistune.Markdown(
renderer=None,
block=TrackingBlockParser(track),
inline=TrackingInlineParser(track),
)
md(md_string)
return track
if __name__ == "__main__":
# In theory we ought to be able to load file this as a Python module
# instead of running it as a script. But this does not work
# because the Python module loading machinery insists that the filename
# must end in .py. But script names ought not to end in .py.
#
# The recipe here
# https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
# does not work with a filename not ending in .py:
# "importlib.util.spec_from_file_location" returns None.
import sys
import json
import argparse
parser = argparse.ArgumentParser(prog="extract-md-links")
parser.add_argument("filename", nargs="?", default="-")
args = parser.parse_args()
if args.filename == "-":
in_file = sys.stdin
else:
in_file = open(args.filename, "r")
text = in_file.read()
print(extract_links(text).as_json())
|
