(perf): use sqlite transactions, and GC less (#847)

We use sqlite transactions to commit changes into the database, rather
than storing all the data in a list before running one big insert.
Hopefully this gives a noticeable perf boost.

We also add `org-roam-db-gc-threshold`, which shaves time by deferring the garbage collection to the end.

CHANGELOG.md

@@ -5,9 +5,11 @@
 ### Breaking Changes
 
 ### Features
-- [#814] Implement `org-roam-insert-immediate`
+- [#814](https://github.com/org-roam/org-roam/pull/814) Implement `org-roam-insert-immediate`
 - [#833](https://github.com/org-roam/org-roam/pull/833) Add customization of file titles with `org-roam-title-to-slug-function`. 
 - [#839](https://github.com/org-roam/org-roam/pull/839) Return selected file from `org-roam-insert`
+- [#847](https://github.com/org-roam/org-roam/pull/847) Add GC threshold `org-roam-db-gc-threshold` to temporarily change the threshold on expensive operations.
+- [#847](https://github.com/org-roam/org-roam/pull/847) Use sqlite3 transactions instead of storing the values to be inserted.
 - [#851](https://github.com/org-roam/org-roam/pull/851) Add `'first-directory'` option for `org-roam-tag-sources`
 
 ### Bugfixes

org-roam-db.el

@@ -60,6 +60,16 @@ when used with multiple Org-roam instances."
   :type 'string
   :group 'org-roam)
 
+(defcustom org-roam-db-gc-threshold most-positive-fixnum
+  "The value to temporarily set the `gc-cons-threshold' threshold to.
+During large, heavy operations like `org-roam-db-build-cache',
+many GC operations happen because of the large number of
+temporary structures generated (e.g. parsed ASTs). Temporarily
+increasing `gc-cons-threshold' will help reduce the number of GC
+operations, at the cost of temporary memory usage."
+  :type 'int
+  :group 'org-roam)
+
 (defconst org-roam-db--version 6)
 
 (defvar org-roam-db--connection (make-hash-table :test #'equal)
@@ -68,7 +78,7 @@ when used with multiple Org-roam instances."
 ;;;; Core Functions
 (defun org-roam-db--get ()
   "Return the sqlite db file."
-    (or org-roam-db-location
+  (or org-roam-db-location
       (expand-file-name "org-roam.db" org-roam-directory)))
 
 (defun org-roam-db--get-connection ()
@@ -403,91 +413,84 @@ If FORCE, force a rebuild of the cache from scratch."
   (when force (delete-file (org-roam-db--get)))
   (org-roam-db--close) ;; Force a reconnect
   (org-roam-db) ;; To initialize the database, no-op if already initialized
-  (let* ((org-roam-files (org-roam--list-all-files))
+  (let* ((gc-cons-threshold org-roam-db-gc-threshold)
+         (org-roam-files (org-roam--list-all-files))
          (current-files (org-roam-db--get-current-files))
-         all-files all-headlines all-links all-titles all-refs all-tags)
+         (file-count 0)
-    ;; Two-step building
+         (headline-count 0)
-    ;; First step: Rebuild files and headlines
+         (link-count 0)
-    (dolist (file org-roam-files)
+         (tag-count 0)
-      (let* ((attr (file-attributes file))
+         (title-count 0)
-             (atime (file-attribute-access-time attr))
+         (ref-count 0)
-             (mtime (file-attribute-modification-time attr)))
+         (deleted-count 0))
+    (emacsql-with-transaction (org-roam-db--get-connection)
+      ;; Two-step building
+      ;; First step: Rebuild files and headlines
+      (dolist (file org-roam-files)
+        (let* ((attr (file-attributes file))
+               (atime (file-attribute-access-time attr))
+               (mtime (file-attribute-modification-time attr)))
+          (org-roam--with-temp-buffer file
+            (let ((contents-hash (secure-hash 'sha1 (current-buffer))))
+              (unless (string= (gethash file current-files)
+                               contents-hash)
+                (org-roam-db--clear-file file)
+                (org-roam-db-query
+                 [:insert :into files
+                  :values $v1]
+                 (vector file contents-hash (list :atime atime :mtime mtime)))
+                (setq file-count (1+ file-count))
+                (when-let (headlines (org-roam--extract-headlines file))
+                  (org-roam-db-query
+                   [:insert :into headlines
+                    :values $v1]
+                   headlines)
+                  (setq headline-count (1+ headline-count))))))))
+      ;; Second step: Rebuild the rest
+      (dolist (file org-roam-files)
         (org-roam--with-temp-buffer file
           (let ((contents-hash (secure-hash 'sha1 (current-buffer))))
             (unless (string= (gethash file current-files)
                              contents-hash)
-              (org-roam-db--clear-file file)
+              (when-let (links (org-roam--extract-links file))
-              (push (vector file contents-hash (list :atime atime :mtime mtime))
+                (org-roam-db-query
-                    all-files)
+                 [:insert :into links
-              (when-let (headlines (org-roam--extract-headlines file))
+                  :values $v1]
-                (push headlines all-headlines)))))))
+                 links)
-    (when all-files
+                (setq link-count (1+ link-count)))
-      (org-roam-db-query
+              (when-let (tags (org-roam--extract-tags file))
-       [:insert :into files
+                (org-roam-db-query
-        :values $v1]
+                 [:insert :into tags
-       all-files))
+                  :values $v1]
-    (when all-headlines
+                 (vector file tags))
-      (org-roam-db-query
+                (setq tag-count (1+ tag-count)))
-       [:insert :into headlines
+              (let ((titles (org-roam--extract-titles)))
-        :values $v1]
+                (org-roam-db-query
-       all-headlines))
+                 [:insert :into titles
-    ;; Second step: Rebuild the rest
+                  :values $v1]
-    (dolist (file org-roam-files)
+                 (vector file titles))
-      (org-roam--with-temp-buffer file
+                (setq title-count (1+ title-count)))
-        (let ((contents-hash (secure-hash 'sha1 (current-buffer))))
+              (when-let* ((ref (org-roam--extract-ref))
-          (unless (string= (gethash file current-files)
+                          (type (car ref))
-                           contents-hash)
+                          (key (cdr ref)))
-            (when-let (links (org-roam--extract-links file))
+                (org-roam-db-query
-              (push links all-links))
+                 [:insert :into refs
-            (when-let (tags (org-roam--extract-tags file))
+                  :values $v1]
-              (push (vector file tags) all-tags))
+                 (vector key file type))
-            (let ((titles (org-roam--extract-titles)))
+                (setq ref-count (1+ ref-count))))
-              (push (vector file titles)
+            (remhash file current-files))))
-                    all-titles))
+      (dolist (file (hash-table-keys current-files))
-            (when-let* ((ref (org-roam--extract-ref))
+        ;; These files are no longer around, remove from cache...
-                        (type (car ref))
+        (org-roam-db--clear-file file)
-                        (key (cdr ref)))
+        (setq deleted-count (1+ deleted-count))))
-              (setq all-refs (cons (vector key file type) all-refs))))
+    (org-roam-message "files: %s, headlines: %s, links: %s, tags: %s, titles: %s, refs: %s, deleted: %s"
-          (remhash file current-files))))
+                      file-count
-    (dolist (file (hash-table-keys current-files))
+                      headline-count
-      ;; These files are no longer around, remove from cache...
+                      link-count
-      (org-roam-db--clear-file file))
+                      tag-count
-    (when all-links
+                      title-count
-      (org-roam-db-query
+                      ref-count
-       [:insert :into links
+                      deleted-count)))
-        :values $v1]
-       all-links))
-    (when all-titles
-      (org-roam-db-query
-       [:insert :into titles
-        :values $v1]
-       all-titles))
-    (when all-tags
-      (org-roam-db-query
-       [:insert :into tags
-        :values $v1]
-       all-tags))
-    (when all-refs
-      (org-roam-db-query
-       [:insert :into refs
-        :values $v1]
-       all-refs))
-    (let ((stats (list :files (length all-files)
-                       :headlines (length all-headlines)
-                       :links (length all-links)
-                       :tags (length all-tags)
-                       :titles (length all-titles)
-                       :refs (length all-refs)
-                       :deleted (length (hash-table-keys current-files)))))
-      (org-roam-message "files: %s, headlines: %s, links: %s, tags: %s, titles: %s, refs: %s, deleted: %s"
-                        (plist-get stats :files)
-                        (plist-get stats :headlines)
-                        (plist-get stats :links)
-                        (plist-get stats :tags)
-                        (plist-get stats :titles)
-                        (plist-get stats :refs)
-                        (plist-get stats :deleted))
-      stats)))
 
 (provide 'org-roam-db)
 

tests/test-org-roam-perf.el

@@ -45,7 +45,7 @@
     (pcase (benchmark-run 1 (org-roam-db-build-cache t))
       (`(,time ,gcs ,time-in-gc)
        (message "Elapsed time: %fs (%fs in %d GCs)" time time-in-gc gcs)
-       (expect time :to-be-less-than 100))))
+       (expect time :to-be-less-than 60))))
   (it "builds quickly without change"
     (pcase (benchmark-run 1 (org-roam-db-build-cache))
       (`(,time ,gcs ,time-in-gc)