1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
#!/usr/bin/env python3
#
# usage:
# maint/cargo-check-publishable
#
# Checks that every crate that is supposed to be published,
# looks like it could be published.
#
# Currently the only check performed is that the `package.categories`
# value is correct for crates.io.
import os
import requests
import sys
import time
from list_crates import list_crates, Crate
from typing import Any, Callable, Dict, TYPE_CHECKING
problems = 0
if not TYPE_CHECKING:
sys.stdout.reconfigure(line_buffering=True)
api_call_memo: Dict[str, Any] = {}
def crates_io_api_call(endpoint: str, response_expect_key: str) -> Any:
'''
Queries
https://crates.io/api/$endpoint
Expects to receive either
HTTP 200 and a json document which is an object containing a key `response_expect_key`
HTTP 404 and a json document containing a `.errors` key
The fetched JSON is returned, or `None` for 404
Reimplementation in Python of the shell version in `crates-io-utils.sh`
'''
try:
base = os.environ['CRATES_IO_URL_BASE']
except KeyError:
base = 'https://crates.io/api'
url = base + '/' + endpoint
try:
json = api_call_memo[url]
except KeyError:
session = requests.Session()
session.headers.update({'User-Agent': 'maint/ scripts for Tor Project CI (python)'})
time.sleep(1)
response = session.get(url)
try:
json = response.json()
except Exception:
print("from url %s obtained %s" % (url, repr(response)), file=sys.stderr)
raise
if response.status_code == 404:
if not 'errors' in json:
raise(RuntimeError('404 but no `errors` URL %s: %s, %s' %
(url, response, repr(json))))
api_call_memo[url] = None
return None
if response.status_code != 200:
raise(RuntimeError('error data from URL %s: %s, %s' %
(url, response, repr(response.content))))
api_call_memo[url] = json
if not response_expect_key in json:
raise(RuntimeError('unexpected JSON data from URL %s: %s' % (url, json)))
return json
def problem(msg: str) -> None:
print('problem: %s' % msg, file=sys.stderr)
global problems
problems += 1
def prepare() -> None:
pass
def check_crate(crate: Crate) -> None:
#if not crate.publish:
# return
print('checking crate %s' % crate.name)
try:
cats = crate.raw_metadata['package']['categories']
except KeyError:
cats = []
for spec_cat in cats:
if '::' in spec_cat:
(cat, subcat) = spec_cat.split('::', 1)
else:
cat = spec_cat
subcat = None
info = crates_io_api_call('v1/categories/%s' % cat, 'category')
if info is None:
problem('category %s not known at crates.io!' % repr(cat))
continue
info = info['category']
if subcat is not None:
if not(any([s['slug'] == spec_cat for s in info['subcategories']])):
problem('subcategory %s not known at crates.io!' % repr(spec_cat))
prepare()
for crate in list_crates():
check_crate(crate)
if problems:
print('%d problems!' % problems, file=sys.stderr)
sys.exit(1)
else:
print('ok!')
|
