Перевод не возобновляем: --all переводит все главы заново, механизм progress не работает
This commit is contained in:
@@ -180,8 +180,16 @@ is the bridge: it writes the repo's input format, shells out to
|
|||||||
|
|
||||||
`python scripts/translate.py book.html book.ru.html --repo <clone> --workdir <dir> --workers 12`
|
`python scripts/translate.py book.html book.ru.html --repo <clone> --workdir <dir> --workers 12`
|
||||||
|
|
||||||
Resumable: the external repo tracks completed chapters and skips them on a
|
⚠️ **Not resumable, despite what the external repo claims.** Its
|
||||||
re-run, so an interrupted run costs nothing to restart.
|
`progress/translation_progress.json` stays `{"chapters": {}}` and `--all`
|
||||||
|
re-translates every chapter, including ones already sitting in
|
||||||
|
`translations/`. Measured 2026-08-21 on Stroustrup: a re-run to repair 4
|
||||||
|
failed chapters re-did all 29. Budget a full book on every restart, and
|
||||||
|
prefer getting one clean run over patching a partial one.
|
||||||
|
|
||||||
|
The one thing that *does* skip work: deleting a chapter from `extracted/`
|
||||||
|
before the run. It is then never sent, and `rebuild` keeps the original
|
||||||
|
English for it — the right treatment for an index.
|
||||||
4. **Read the reported counts.** "без перевода" above zero means a chapter came
|
4. **Read the reported counts.** "без перевода" above zero means a chapter came
|
||||||
back with a different paragraph count and kept its original text; "разметка
|
back with a different paragraph count and kept its original text; "разметка
|
||||||
потеряна" counts paragraphs where the model mangled the inline-tag markers
|
потеряна" counts paragraphs where the model mangled the inline-tag markers
|
||||||
|
|||||||
@@ -105,6 +105,33 @@ def test_code_listings_are_left_alone(tmp: Path):
|
|||||||
assert t.verify_output(src, "ru"), "код не должен заваливать приёмку"
|
assert t.verify_output(src, "ru"), "код не должен заваливать приёмку"
|
||||||
|
|
||||||
|
|
||||||
|
def test_junk_translation_falls_back_to_original(tmp: Path):
|
||||||
|
"""Модель иногда возвращает на длинный абзац огрызок «, v», а число абзацев
|
||||||
|
при этом сходится — прежние ворота такое пропускали."""
|
||||||
|
src = tmp / "book.html"
|
||||||
|
long_en = ("A vector is simply a sequence of elements that you can access by "
|
||||||
|
"an index, and it is the workhorse of the standard library. ") * 2
|
||||||
|
src.write_text("\n".join([
|
||||||
|
"<h1>Chapter</h1>",
|
||||||
|
"<p>%s</p>" % long_en,
|
||||||
|
"<p>Second paragraph of the very same chapter, also reasonably long.</p>",
|
||||||
|
]), encoding="utf-8")
|
||||||
|
chapters = t.split_chapters(t.parse_blocks(src))
|
||||||
|
trans = tmp / "translations"
|
||||||
|
trans.mkdir()
|
||||||
|
(trans / "chapter_000_translated.json").write_text(json.dumps(
|
||||||
|
{"number": 0, "paragraphs": ["Глава", ", v",
|
||||||
|
"Второй абзац той же самой главы, тоже достаточно длинный."]},
|
||||||
|
ensure_ascii=False), encoding="utf-8")
|
||||||
|
|
||||||
|
out = tmp / "out.html"
|
||||||
|
t.rebuild(src, chapters, tmp, out)
|
||||||
|
result = out.read_text(encoding="utf-8")
|
||||||
|
assert ", v" not in result, "огрызок не должен попадать в книгу"
|
||||||
|
assert "A vector is simply a sequence" in result, "вместо огрызка нужен оригинал"
|
||||||
|
assert "Второй абзац" in result, "нормальный перевод должен остаться"
|
||||||
|
|
||||||
|
|
||||||
def test_workdir_belongs_to_one_book(tmp: Path):
|
def test_workdir_belongs_to_one_book(tmp: Path):
|
||||||
"""Имена chapter_NNN.json у всех книг одинаковы: чужой workdir склеит
|
"""Имена chapter_NNN.json у всех книг одинаковы: чужой workdir склеит
|
||||||
перевод одной книги с текстом другой."""
|
перевод одной книги с текстом другой."""
|
||||||
@@ -155,6 +182,7 @@ if __name__ == "__main__":
|
|||||||
test_broken_marks_drop_tags()
|
test_broken_marks_drop_tags()
|
||||||
test_language_detection()
|
test_language_detection()
|
||||||
for case in (test_split_and_rebuild, test_code_listings_are_left_alone,
|
for case in (test_split_and_rebuild, test_code_listings_are_left_alone,
|
||||||
|
test_junk_translation_falls_back_to_original,
|
||||||
test_workdir_belongs_to_one_book, test_stale_chapters_removed,
|
test_workdir_belongs_to_one_book, test_stale_chapters_removed,
|
||||||
test_verify_output_catches_untranslated):
|
test_verify_output_catches_untranslated):
|
||||||
with tempfile.TemporaryDirectory() as d:
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
|||||||
+16
-5
@@ -148,9 +148,15 @@ def run_translator(repo, workdir, extracted, workers):
|
|||||||
sys.exit("book_translator завершился с кодом %d" % r.returncode)
|
sys.exit("book_translator завершился с кодом %d" % r.returncode)
|
||||||
|
|
||||||
|
|
||||||
|
# Доля от длины оригинала, ниже которой перевод считается мусором. Совпадение
|
||||||
|
# числа абзацев ничего не гарантирует: модель иногда возвращает на длинный абзац
|
||||||
|
# огрызок вида «, v», и прежние ворота такое пропускали.
|
||||||
|
MIN_LEN_SHARE = 0.25
|
||||||
|
|
||||||
|
|
||||||
def rebuild(src, chapters, workdir, out):
|
def rebuild(src, chapters, workdir, out):
|
||||||
lines = src.read_text(encoding="utf-8").splitlines()
|
lines = src.read_text(encoding="utf-8").splitlines()
|
||||||
broken = missing = translated = 0
|
broken = missing = translated = junk = 0
|
||||||
for n, ch in enumerate(chapters):
|
for n, ch in enumerate(chapters):
|
||||||
f = workdir / "translations" / ("chapter_%03d_translated.json" % n)
|
f = workdir / "translations" / ("chapter_%03d_translated.json" % n)
|
||||||
if not f.exists():
|
if not f.exists():
|
||||||
@@ -162,15 +168,20 @@ def rebuild(src, chapters, workdir, out):
|
|||||||
% (n, len(paragraphs), len(ch)))
|
% (n, len(paragraphs), len(ch)))
|
||||||
missing += len(ch)
|
missing += len(ch)
|
||||||
continue
|
continue
|
||||||
for (idx, tag, attrs, _), text in zip(ch, paragraphs):
|
for (idx, tag, attrs, original), text in zip(ch, paragraphs):
|
||||||
|
plain_src = re.sub("<[^>]+>", "", original)
|
||||||
|
if len(plain_src) > 120 and len(text) < max(20, len(plain_src) * MIN_LEN_SHARE):
|
||||||
|
junk += 1
|
||||||
|
lines[idx] = "<%s%s>%s</%s>" % (tag, attrs, original, tag)
|
||||||
|
continue # оставляем оригинал: английский абзац лучше огрызка
|
||||||
body, ok = from_marks(text)
|
body, ok = from_marks(text)
|
||||||
broken += not ok
|
broken += not ok
|
||||||
translated += 1
|
translated += 1
|
||||||
lines[idx] = "<%s%s>%s</%s>" % (tag, attrs, body, tag)
|
lines[idx] = "<%s%s>%s</%s>" % (tag, attrs, body, tag)
|
||||||
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
out.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||||
print("переведено блоков: %d, без перевода: %d, разметка потеряна в %d"
|
print("переведено блоков: %d, без перевода: %d, разметка потеряна в %d, "
|
||||||
% (translated, missing, broken))
|
"огрызков заменено оригиналом: %d" % (translated, missing, broken, junk))
|
||||||
return missing == 0
|
return missing == 0 and junk * 200 <= translated
|
||||||
|
|
||||||
|
|
||||||
def verify_output(out, target):
|
def verify_output(out, target):
|
||||||
|
|||||||
Reference in New Issue
Block a user