From 06d782e77aa38b4dc007ea4d21351f582a31fadf Mon Sep 17 00:00:00 2001 From: xenofem Date: Sat, 8 Jun 2024 07:20:44 -0400 Subject: [PATCH] much more efficient caching for suggestions --- dlibrary/dlibrary.py | 129 +++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 47 deletions(-) diff --git a/dlibrary/dlibrary.py b/dlibrary/dlibrary.py index cb6cb0e..c963ee1 100755 --- a/dlibrary/dlibrary.py +++ b/dlibrary/dlibrary.py @@ -1150,7 +1150,7 @@ def copy_recursive(src, dest): memoized_similarities = {} -def similarity(a, b, cache_cur=None): +def string_similarity(a, b): if len(a) < len(b) or (len(a) == len(b) and a < b): shorter = a longer = b @@ -1163,43 +1163,49 @@ def similarity(a, b, cache_cur=None): if (shorter, longer) in memoized_similarities: return memoized_similarities[(shorter, longer)] - if cache_cur and (cached := cache_cur.execute("SELECT similarity FROM similarities WHERE shorter = ? AND longer = ?", (shorter, longer)).fetchone()) is not None: - result = cached[0] - else: - options = [similarity(shorter[1:], longer)] - for i in range(1, len(shorter)+1): - match_idx = longer.find(shorter[:i]) - if match_idx == -1: - break - options.append(i*i + similarity(shorter[i:], longer[match_idx+i:])) - result = max(options) - - if cache_cur: - cache_cur.execute( - "INSERT INTO similarities(shorter, longer, similarity) VALUES(?, ?, ?)", - (shorter, longer, result), - ) + options = [string_similarity(shorter[1:], longer)] + for i in range(1, len(shorter)+1): + match_idx = longer.find(shorter[:i]) + if match_idx == -1: + break + options.append(i*i + string_similarity(shorter[i:], longer[match_idx+i:])) + result = max(options) memoized_similarities[(shorter, longer)] = result return result -def top(items, n, key, overflow=0): - winners = [] - for item in items: - score = key(item) - if len(winners) < n or score >= winners[-1][1]: - for i in range(len(winners) + 1): - if i == len(winners) or score >= winners[i][1]: - winners.insert(i, (item, score)) - break - while len(winners) > n and winners[-1][1] < winners[n-1][1]: - winners.pop() +class TopScoreList: + def __init__(self, limit): + self.limit = limit + self.items_with_scores = [] + self.randomized = True - # shuffle followed by stable sort to randomly shuffle within each score tier - random.shuffle(winners) - winners.sort(key=lambda w: w[1], reverse=True) + def insert(self, item, score): + if len(self.items_with_scores) >= self.limit and score < self.items_with_scores[-1][1]: + return [item] - return [item for (item, score) in winners[:n+overflow]] + self.randomized = False + for i in range(len(self.items_with_scores) + 1): + if i == len(self.items_with_scores) or score >= self.items_with_scores[i][1]: + self.items_with_scores.insert(i, (item, score)) + break + removed_items = [] + while len(self.items_with_scores) > self.limit and self.items_with_scores[-1][1] < self.items_with_scores[self.limit-1][1]: + removed_items.append(self.items_with_scores.pop()[0]) + return removed_items + + def _randomize(self): + if self.randomized: + return + + # shuffle followed by stable sort to randomly shuffle within each score tier + random.shuffle(self.items_with_scores) + self.items_with_scores.sort(key=lambda i: i[1], reverse=True) + self.randomized = True + + def __iter__(self): + self._randomize() + return (item for (item, _) in self.items_with_scores[:self.limit]) def generate(args): debug('loading templates') @@ -1219,17 +1225,21 @@ def generate(args): cur = con.cursor() debug('main database open') - debug('opening cache database') + debug('opening suggestion cache database') cache_con = sqlite3.connect(args.destdir / 'cache.db') cache_cur = cache_con.cursor() - cache_cur.execute("CREATE TABLE IF NOT EXISTS similarities(shorter TEXT, longer TEXT, similarity INT, PRIMARY KEY(shorter, longer))") - debug('cache database open') + cache_cur.execute("CREATE TABLE IF NOT EXISTS suggestions(work TEXT, suggested TEXT, similarity INT, PRIMARY KEY(work, suggested))") + debug('suggestion cache database open') + cached_suggestions = {} + for (work, suggested, similarity) in cache_cur.execute('SELECT work, suggested, similarity FROM suggestions'): + cached_suggestions.setdefault(work, TopScoreList(SUGGESTED_WORKS_COUNT)).insert(suggested, similarity) + debug('cached suggestions loaded') site_dir = args.destdir / 'site' collated_work_ids = {p.name for p in (site_dir / 'images').iterdir()} - works = [] + works = {} debug('checking thumbnail files') thumbnail_files = {f.stem: f for f in (site_dir / 'thumbnails').iterdir()} debug(f'{len(thumbnail_files)} thumbnail files found') @@ -1256,30 +1266,55 @@ def generate(args): 'thumbnail_path': thumbnail_path, 'images': images, } - works.append(work) + works[work_id] = work print(f'{ANSI_LINECLEAR}{idx+1} database entries read...', end='') print() - for (idx, work) in enumerate(works): - def suggestion_priority(other_work): + for work in works.values(): + if work['id'] in cached_suggestions: + continue + debug(f'Computing suggestions for new work {work["title"]}') + cached_suggestions[work['id']] = TopScoreList(SUGGESTED_WORKS_COUNT) + for other_work in works.values(): if other_work is work: - return -2 + continue if work['series'] and work['series'] == other_work['series']: - return -1 - return similarity(work['title'], other_work['title'], cache_cur) - suggested = top(works, SUGGESTED_WORKS_COUNT, suggestion_priority) + continue + if other_work['id'] not in cached_suggestions: + continue # we'll get to it later + similarity = string_similarity(work['title'], other_work['title']) + cached_suggestions[work['id']].insert(other_work['id'], similarity) + removed = cached_suggestions[other_work['id']].insert(work['id'], similarity) + if removed != [work['id']]: + cache_cur.executemany( + 'DELETE FROM suggestions WHERE work = :work AND suggested = :suggested', + [{ "work": other_work['id'], "suggested": item } for item in removed], + ) + cache_cur.execute( + 'INSERT INTO suggestions(work, suggested, similarity) VALUES(:work, :suggested, :similarity)', + { "work": other_work['id'], "suggested": work['id'], "similarity": similarity }, + ) + cache_cur.executemany( + 'INSERT INTO suggestions(work, suggested, similarity) VALUES(:work, :suggested, :similarity)', + [{ "work": work['id'], "suggested": suggested, "similarity": similarity } for (suggested, similarity) in cached_suggestions[work['id']].items_with_scores], + ) + cache_con.commit() + + for (idx, work) in enumerate(works.values()): work_dir = site_dir / 'works' / work['id'] viewer_dir = work_dir / 'view' viewer_dir.mkdir(parents=True, exist_ok=True) with open(work_dir / 'index.html', 'w') as f: - f.write(work_template.render(depth=2, work=work, title=work['title'], suggested=suggested)) + f.write(work_template.render( + depth=2, work=work, title=work['title'], + suggested=[works[suggested] for suggested in cached_suggestions[work['id']]], + )) with open(viewer_dir / 'index.html', 'w') as f: f.write(viewer_template.render(depth=3, work=work, title=work['title'])) count_progress(idx, len(works), 'works processed') - cache_con.commit() uca = pyuca.Collator().sort_key def make_categorization(categorization, query, work_filter, work_style_cards=False): @@ -1288,7 +1323,7 @@ def generate(args): cats = sorted((cat for (cat,) in cur.execute(query)), key=uca) cat_samples = {} for (idx, cat) in enumerate(cats): - cat_works = list(filter(work_filter(cat), works)) + cat_works = list(filter(work_filter(cat), works.values())) cat_samples[cat] = cat_works[0] if len(cat_works) > 0 else None safeish_cat = cat.replace('/', ' ') @@ -1342,7 +1377,7 @@ def generate(args): debug('writing index page') with open(site_dir / 'index.html', 'w') as f: - f.write(index_template.render(depth=0, works=works)) + f.write(index_template.render(depth=0, works=list(works.values()))) debug('index page written') debug('closing cache database')