(experimental): react to directory related events in the database

With filenotify events related to subdirectories, automatically
invalidate no longer valid entries and add the new ones as they appear.

org-roam-capture.el

@@ -6,7 +6,7 @@
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
 ;; Version: 2.0.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1") (filenotify-recursive "0.0.1"))
 
 ;; This file is NOT part of GNU Emacs.
 

org-roam-compat.el

@@ -163,6 +163,9 @@ nodes." org-id-locations-file)
 (define-obsolete-function-alias
   'org-roam-teardown
   'org-roam-db-autosync-disable "org-roam 2.0")
+(define-obsolete-variable-alias
+  'org-roam-db-update-on-save
+  'org-roam-db-autosync-update-method "org-roam 2.0")
 
 (define-obsolete-variable-alias
   'org-roam-current-node

org-roam-db.el

@@ -6,7 +6,7 @@
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
 ;; Version: 2.0.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1") (filenotify-recursive "0.0.1"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -31,6 +31,9 @@
 ;;
 ;;; Code:
 (require 'org-roam)
+(require 'filenotify)
+(require 'filenotify-recursive)
+
 (defvar org-outline-path-cache)
 
 ;;; Options
@@ -71,26 +74,38 @@ database."
   :type 'function
   :group 'org-roam)
 
-(defcustom org-roam-db-update-on-save t
+(defcustom org-roam-db-autosync-update-method
-  "If t, update the Org-roam database upon saving the file.
+  (if file-notify--library 'filenotify 'on-save)
-Disable this if your files are large and updating the database is
+  "What method to use to keep Org-roam's database updated.
-slow."
+
-  :type 'boolean
+  'filenotify
+    Update Org-roam upon detecting changes from the filesystem using
+    file watchers. Requires Emacs that's compiled with support for
+    file notifications.
+
+  'on-save
+    Update the database whenever Emacs buffer that visits an Org-roam
+    file is saved. Unlike `filenotify' this won't be able to react to
+    external changes in the filesystem.
+
+  nil
+    Do not automatically update the Org-roam database."
+  :type '(choice (const :tag "Filenotify" filenotify)
+                 (const :tag "On save" onsave)
+                 (const :tag "Do not autoupdate" nil))
   :group 'org-roam)
 
-;;; Variables
+;;; Initialization
 (defconst org-roam-db-version 16)
 
-;; TODO Rename this
+(defvar org-roam-db--connection (make-hash-table :test #'equal)
+  "Database connection to Org-roam database.")
+
 (defconst org-roam--sqlite-available-p
   (with-demoted-errors "Org-roam initialization: %S"
     (emacsql-sqlite-ensure-binary)
     t))
 
-(defvar org-roam-db--connection (make-hash-table :test #'equal)
-  "Database connection to Org-roam database.")
-
-;;; Core Functions
 (defun org-roam-db--get-connection ()
   "Return the database connection, if any."
   (gethash (expand-file-name org-roam-directory)
@@ -474,6 +489,8 @@ If UPDATE-P is non-nil, first remove the file in the database."
 If the file does not exist anymore, remove it from the cache.
 If the file exists, update the cache with information."
   (setq file-path (or file-path (buffer-file-name (buffer-base-buffer))))
+  (org-roam-db-clear-file file-path)
+  (when (file-exists-p file-path)
     (let ((content-hash (org-roam-db--file-hash file-path))
           (db-hash (caar (org-roam-db-query [:select hash :from files
                                              :where (= file $s1)] file-path))))
@@ -481,7 +498,6 @@ If the file exists, update the cache with information."
         (org-roam-with-file file-path nil
           (save-excursion
             (org-set-regexps-and-options 'tags-only)
-          (org-roam-db-clear-file)
             (org-roam-db-insert-file)
             (org-roam-db-insert-file-node)
             (setq org-outline-path-cache nil)
@@ -492,7 +508,7 @@ If the file exists, update the cache with information."
                    #'org-roam-db-insert-refs))
             (setq org-outline-path-cache nil)
             (org-roam-db-map-links
-           (list #'org-roam-db-insert-link)))))))
+             (list #'org-roam-db-insert-link))))))))
 
 ;;;###autoload
 (defun org-roam-db-sync (&optional force)
@@ -527,6 +543,19 @@ If FORCE, force a rebuild of the cache from scratch."
         (dolist (file modified-files)
           (org-roam-db-update-file file))))))
 
+;;;###autoload
+(defun org-roam-db-autosync-enable ()
+  "Activate `org-roam-db-autosync-mode'."
+  (org-roam-db-autosync-mode +1))
+
+(defun org-roam-db-autosync-disable ()
+  "Deactivate `org-roam-db-autosync-mode'."
+  (org-roam-db-autosync-mode -1))
+
+(defun org-roam-db-autosync-toggle ()
+  "Toggle `org-roam-db-autosync-mode' enabled/disabled."
+  (org-roam-db-autosync-mode 'toggle))
+
 ;;;###autoload
 (define-minor-mode org-roam-db-autosync-mode
   "Global minor mode to keep your Org-roam session automatically synchronized.
@@ -540,37 +569,101 @@ database, see `org-roam-db-sync' command."
   :group 'org-roam
   :global t
   :init-value nil
-  (let ((enabled org-roam-db-autosync-mode))
+  (let ((enabled org-roam-db-autosync-mode)
+        (update-method org-roam-db-autosync-update-method))
     (cond
      (enabled
       (add-hook 'find-file-hook  #'org-roam-db-autosync--setup-file-h)
+      (org-roam-db-autosync--update-method :enable update-method)
       (add-hook 'kill-emacs-hook #'org-roam-db--close-all)
-      (advice-add #'rename-file :after  #'org-roam-db-autosync--rename-file-a)
-      (advice-add #'delete-file :before #'org-roam-db-autosync--delete-file-a)
       (org-roam-db-sync))
      (t
       (remove-hook 'find-file-hook  #'org-roam-db-autosync--setup-file-h)
+      (org-roam-db-autosync--update-method :disable update-method)
       (remove-hook 'kill-emacs-hook #'org-roam-db--close-all)
-      (advice-remove #'rename-file #'org-roam-db-autosync--rename-file-a)
-      (advice-remove #'delete-file #'org-roam-db-autosync--delete-file-a)
       (org-roam-db--close-all)
       ;; Disable local hooks for all org-roam buffers
       (dolist (buf (org-roam-buffer-list))
         (with-current-buffer buf
           (remove-hook 'after-save-hook #'org-roam-db-autosync--try-update-on-save-h t)))))))
 
-;;;###autoload
+(defvar org-roam-db-autosync--filenotify-descriptors (list)
-(defun org-roam-db-autosync-enable ()
+  "An alist mapping watched Org-roam directories to `filenotify-recursive' uuid.")
-  "Activate `org-roam-db-autosync-mode'."
-  (org-roam-db-autosync-mode +1))
 
-(defun org-roam-db-autosync-disable ()
+(defun org-roam-db-autosync--update-method (state method)
-  "Deactivate `org-roam-db-autosync-mode'."
+  "Change the current `org-roam-db-autosync-update-method' to METHOD.
-  (org-roam-db-autosync-mode -1))
+STATE should be either :enable or :disable, while METHOD should
+be on of the values from `org-roam-db-autosync-update-method'."
+  (unless (memq method '(filenotify on-save nil))
+    (user-error "Unknown `org-roam-db-autosync-update-method': %s" method))
+  (cl-ecase state
+    (:enable
+     (unless (eq method org-roam-db-autosync-update-method)
+       ;; Clean up the old method in case of hot swap.
+       (org-roam-db-autosync--update-method :disable org-roam-db-autosync-update-method))
+     (setq org-roam-db-autosync-update-method method)
+     (pcase method
+       ('filenotify
+        (cl-pushnew
+         (cons org-roam-directory
+               (fnr-add-watch org-roam-directory
+                              '(change)
+                              #'org-roam-db-autosync--filenotify-update
+                              "\\`\\.")) ; Ignore directories that start with "."
+         org-roam-db-autosync--filenotify-descriptors))
+       ('on-save
+        (add-hook 'org-roam-find-file-hook #'org-roam-db-autosync--setup-update-on-save-h)
+        (advice-add #'rename-file :after  #'org-roam-db-autosync--rename-file-a)
+        (advice-add #'delete-file :before #'org-roam-db-autosync--delete-file-a))
+       ((pred nilp)
+        t)))
+    (:disable
+     (pcase org-roam-db-autosync-update-method
+       ('filenotify
+        (cl-loop for entry in org-roam-db-autosync--filenotify-descriptors
+                 for _dir = (car entry)
+                 for uuid = (cdr entry)
+                 do (fnr-rm-watch uuid))
+        (setq org-roam-db-autosync--filenotify-descriptors nil))
+       ('on-save
+        (remove-hook 'org-roam-find-file-hook #'org-roam-db-autosync--setup-update-on-save-h)
+        (advice-remove #'rename-file #'org-roam-db-autosync--rename-file-a)
+        (advice-remove #'delete-file #'org-roam-db-autosync--delete-file-a))
+       ((pred nilp)
+        t)))))
 
-(defun org-roam-db-autosync-toggle ()
+(defun org-roam-db-autosync--filenotify-update (event)
-  "Toggle `org-roam-db-autosync-mode' enabled/disabled."
+  "Update Org-roam's database according to EVENT sent by `filenotify'."
-  (org-roam-db-autosync-mode 'toggle))
+  (cl-destructuring-bind (_descriptor action &rest files) event
+    (cond
+     ((cl-find-if #'org-roam-file-p files)
+      (mapc #'org-roam-db-update-file files))
+     ((memq action '(created deleted renamed))
+      (apply (intern (format "org-roam-db-autosync--update-%s-dir" action)) files)))))
+
+(defun org-roam-db-autosync--update-created-dir (dir)
+  "Add entries from Org-roam files under DIR to the database."
+  (when (file-directory-p dir)
+    (let ((files (let ((org-roam-directory dir))
+                   (org-roam-list-files))))
+      (emacsql-with-transaction (org-roam-db)
+        (mapc #'org-roam-db-update-file files)))))
+
+(defun org-roam-db-autosync--update-deleted-dir (dir)
+  "Invalidate entries related to Org-roam files under DIR from the database."
+  (let ((dir (thread-first dir
+               ;; Ensure that separator is present in the name
+               (directory-file-name)
+               (concat (f-path-separator))
+               ;; Follow the same format as the rest of the files in the database
+               (expand-file-name))))
+    (org-roam-db-query [:delete :from files :where (like file $s1)]
+                       (concat dir "%"))))
+
+(defun org-roam-db-autosync--update-renamed-dir (old-name new-name)
+  "Invalidate and then add files renamed from OLD-NAME directory to NEW-NAME."
+  (org-roam-db-autosync--update-deleted-dir old-name)
+  (org-roam-db-autosync--update-created-dir new-name))
 
 (defun org-roam-db-autosync--delete-file-a (file &optional _trash)
   "Maintain cache consistency when file deletes.
@@ -601,14 +694,9 @@ OLD-FILE is cleared from the database, and NEW-FILE-OR-DIR is added."
   "Setup the current buffer if it visits an Org-roam file."
   (when (org-roam-file-p) (run-hooks 'org-roam-find-file-hook)))
 
-(add-hook 'org-roam-find-file-hook #'org-roam-db-autosync--setup-update-on-save-h)
 (defun org-roam-db-autosync--setup-update-on-save-h ()
   "Setup the current buffer to update the DB after saving the current file."
-  (add-hook 'after-save-hook #'org-roam-db-autosync--try-update-on-save-h nil t))
+  (add-hook 'after-save-hook #'org-roam-db-update-file nil t))
-
-(defun org-roam-db-autosync--try-update-on-save-h ()
-  "If appropriate, update the database for the current file after saving buffer."
-  (when org-roam-db-update-on-save (org-roam-db-update-file)))
 
 ;;; Diagnostics
 (defun org-roam-db-diagnose-node ()

org-roam-migrate.el

@@ -6,7 +6,7 @@
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
 ;; Version: 2.0.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1") (filenotify-recursive "0.0.1"))
 
 ;; This file is NOT part of GNU Emacs.
 

org-roam-mode.el

@@ -6,7 +6,7 @@
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
 ;; Version: 2.0.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1") (filenotify-recursive "0.0.1"))
 
 ;; This file is NOT part of GNU Emacs.
 

org-roam.el

@@ -6,7 +6,7 @@
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
 ;; Version: 2.0.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1") (filenotify-recursive "0.0.1"))
 
 ;; This file is NOT part of GNU Emacs.