From 11ea5a0d58fff6cfb94a2c49db65e31ec4ec42c1 Mon Sep 17 00:00:00 2001
From: xenofem <xenofem@xeno.science>
Date: Mon, 11 Mar 2024 05:47:47 -0400
Subject: [PATCH 1/2] more useful explanations of PDF failures

---
 dlibrary/dlibrary.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dlibrary/dlibrary.py b/dlibrary/dlibrary.py
index 25cccb5..38ece1d 100755
--- a/dlibrary/dlibrary.py
+++ b/dlibrary/dlibrary.py
@@ -316,7 +316,7 @@ class Collator:
         with fitz.open(src) as pdf:
             images = pdf_images(pdf, self.args.force_convert_pdf)
             if images is None:
-                print(f'Support for weirder PDFs not yet implemented, skipping {src}')
+                print(f'Failed to enumerate page images in PDF, skipping {src}')
                 return None
 
             self.dest.mkdir(parents=True, exist_ok=True)
@@ -466,10 +466,11 @@ def pdf_images(pdf, force=False):
                 else:
                     yield extract_image(pdf, xref)
             else:
+                print(f'\nPage {idx+1}: {len(page_images)} images, {len([img for img in page_images if img["xref"] == 0])} non-xref images, {len(page.get_text("blocks"))} total objects')
                 if xref_mode:
                     raise ValueError
                 else:
-                    print(f'\nGenerating pixmap for page {idx+1}')
+                    print(f'Generating pixmap for page {idx+1}')
                     pix = page.get_pixmap(dpi=PDF_FALLBACK_DPI)
                     yield { 'ext': 'png', 'image': pix.tobytes('png') }
             print(f'\x1b[2K\r{idx+1}/{pdf.page_count} pages processed...', end='')

From 6b3982ecf0b450345b68e7eaa8ddff09998a4a45 Mon Sep 17 00:00:00 2001
From: xenofem <xenofem@xeno.science>
Date: Mon, 11 Mar 2024 12:08:08 -0400
Subject: [PATCH 2/2] don't get confused by irrelevant bits of text inserted by
 PDF generation tools

---
 dlibrary/dlibrary.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/dlibrary/dlibrary.py b/dlibrary/dlibrary.py
index 38ece1d..c246cc9 100755
--- a/dlibrary/dlibrary.py
+++ b/dlibrary/dlibrary.py
@@ -79,6 +79,8 @@ IGNOREABLE_EXTENSIONS = ['.txt', '.html', '.htm', '.psd', '.mp4']
 
 PDF_FALLBACK_DPI = 300
 
+IRRELEVANT_PDF_BLOCK_REGEX = re.compile(r'\bTCPDF\b', re.I)
+
 def open_zipfile_with_encoding(path):
     try:
         return zipfile.ZipFile(path, metadata_encoding="utf-8")
@@ -437,9 +439,22 @@ class Collator:
 
         return self.collate_from_paths(srcs_matching_language)
 
-def is_single_image(page):
+def block_is_image(block):
+    return block[6] == 1
+
+def block_text(block):
+    return block[4]
+
+def block_relevant(block):
+    return block_is_image(block) or not IRRELEVANT_PDF_BLOCK_REGEX.search(block_text(block))
+
+def relevant_blocks(page):
     blocks = page.get_text('blocks')
-    return len(blocks) == 1 and blocks[0][6] == 1
+    return [block for block in blocks if block_relevant(block)]
+
+def is_single_image(page):
+    blocks = relevant_blocks(page)
+    return len(blocks) == 1 and block_is_image(blocks[0])
 
 def extract_image(pdf, xref):
     image = pdf.extract_image(xref)
@@ -466,7 +481,7 @@ def pdf_images(pdf, force=False):
                 else:
                     yield extract_image(pdf, xref)
             else:
-                print(f'\nPage {idx+1}: {len(page_images)} images, {len([img for img in page_images if img["xref"] == 0])} non-xref images, {len(page.get_text("blocks"))} total objects')
+                print(f'\nPage {idx+1}: {len(page_images)} images, {len([img for img in page_images if img["xref"] == 0])} non-xref images, {len(relevant_blocks(page))} total relevant objects')
                 if xref_mode:
                     raise ValueError
                 else: