1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
#!/usr/bin/env python3
# ----------
# Import everything we need
import importlib.metadata
import os
import re
import subprocess
import sys
from pathlib import Path
from typing import Union
MISSING_IMPORTS = []
try:
import flake8.main.cli # type: ignore
except ModuleNotFoundError:
MISSING_IMPORTS.append("flake8")
try:
import mypy.main
except ModuleNotFoundError:
MISSING_IMPORTS.append("mypy")
try:
import black
except ModuleNotFoundError:
MISSING_IMPORTS.append("black")
def check_pkg(pkg: str):
try:
__import__(pkg)
return
except ModuleNotFoundError:
MISSING_IMPORTS.append(pkg)
def check_meta(pkg: str):
try:
importlib.metadata.version(pkg)
except importlib.metadata.PackageNotFoundError:
MISSING_IMPORTS.append(pkg)
# We don't use these packages, but our dependencies do.
check_pkg("tomli_w")
check_pkg("marko")
# mypy needs the type annotations in these packages.
check_meta("types-toml")
check_meta("types-PyYAML")
check_meta("types-beautifulsoup4")
check_meta("types-requests")
if MISSING_IMPORTS:
missing = ", ".join(MISSING_IMPORTS)
print(f"Missing packages: Please install {missing}", file=sys.stderr)
sys.exit(1)
# ----------
# Find the things that we want to format or test.
TOPLEVEL = Path(__file__).resolve().parent.parent
os.chdir(TOPLEVEL)
# Scripts are files that start with a python shebang
PYTHON_SHEBANG = re.compile(r"^#!\s*/usr/bin/env\s+python.*")
def starts_with_shebang(path: Path) -> bool:
"""
Return true if `path` is a file representing an independent
python script.
"""
try:
with open(path, "r") as f:
line1 = f.readline()
return PYTHON_SHEBANG.match(line1) is not None
except UnicodeDecodeError:
return False
class Files:
"""List of files in the arti repository"""
files: set[Path]
def __init__(self):
# Requires that we're in the top level of the repository.
# (We always chdir there before initializing this class.)
output = subprocess.run(
["git", "ls-tree", "-r", "--name-only", "HEAD"],
capture_output=True,
encoding="utf-8",
check=True,
)
self.files = [Path(p) for p in output.stdout.split()]
def package_roots(self) -> list[Path]:
"""Find every directory that is the root of a package."""
return [p.parent for p in self.files if p.name == "pyproject.toml"]
def scripts(self) -> list[Path]:
"""
Return every python script, including those inside a package.
(For here, that's defined as a file starting with something like
`#!/usr/bin/env python3`)
"""
return [p for p in self.files if p.is_file() and starts_with_shebang(p)]
def other_python(self) -> list[Path]:
"""
Return every .py file that is not in a package.
"""
packages = self.package_roots()
return [
p
for p in self.files
if p.is_file()
and p.name.endswith(".py")
and not any(p.is_relative_to(pkg) for pkg in packages)
]
def argify(args) -> list[str]:
"""Convert args to a list of strings"""
return list(str(s) for s in args)
# TODO: Find a mypy annotation that works for `targets` here.
def run_flake8_ok(targets) -> bool:
exit_code = flake8.main.cli.main(
argify(["--config", TOPLEVEL / ".flake8"] + targets)
)
return exit_code == 0
def run_mypy_ok(targets, strict=False) -> bool:
exitcode: Union[None, int, str] = 0
if strict:
flags = ["--strict"]
else:
flags = []
try:
mypy.main.main(args=argify(flags + targets), clean_exit=True)
except SystemExit as e:
exitcode = e.code
return exitcode in [0, None]
def run_black_ok(targets) -> bool:
exitcode: Union[None, int, str] = 0
try:
black.main(["--check"] + argify(targets))
except SystemExit as e:
exitcode = e.code
return exitcode in [0, None]
# Some of our scripts require extra dependencies;
# we list them here.
EXTRA_DEPS: dict[Path, list[Path]] = {
Path("maint/rpc-docs-tool"): [Path("python/arti_rpc")]
}
# Some of our scripts support strict type-checking;
# we list them here.
#
# TODO: Get more things to pass with "--strict", and then turn this
# into a _not_ strict list.
MYPY_STRICT = set(
Path(p)
for p in [
"maint/update-md-links",
"maint/list_crates",
"maint/cargo-check-publishable",
]
)
FILES = Files()
SCRIPTS = FILES.scripts()
OTHER_PYTHON = FILES.other_python()
PACKAGES = FILES.package_roots()
okay = True
okay &= run_flake8_ok(SCRIPTS + OTHER_PYTHON + PACKAGES)
# We need to type-check these items separately, or mypy complains
# about too many modules called "__main__".
okay &= run_mypy_ok(PACKAGES)
for item in SCRIPTS:
args = [item]
if item not in OTHER_PYTHON:
args += OTHER_PYTHON
args += EXTRA_DEPS.get(item, [])
okay &= run_mypy_ok(args, strict=item in MYPY_STRICT)
okay &= run_black_ok(SCRIPTS + OTHER_PYTHON + PACKAGES)
if okay:
print("No warnings!")
sys.exit(0)
else:
print("WARNINGS FOUND.")
sys.exit(1)
|
