v2.2.2 (#2171)

Create initial log module for working with Org logs

.dir-locals.el

@@ -4,10 +4,13 @@
   (elisp-lint-ignored-validators . ("byte-compile" "package-lint"))
   (elisp-lint-indent-specs . ((describe . 1)
                               (it . 1)
+                              (thread-first . 0)
+                              (cl-flet . 1)
+                              (cl-flet* . 1)
                               (org-element-map . defun)
+                              (org-roam-dolist-with-progress . 2)
                               (org-roam-with-temp-buffer . 1)
                               (org-with-point-at . 1)
                               (magit-insert-section . defun)
                               (magit-section-case . 0)
-                              (->> . 1)
                               (org-roam-with-file . 2)))))

.github/FUNDING.yml

@@ -1,6 +1,6 @@
 # These are supported funding model platforms
 
-github: [jethrokuan, zaeph]
+github: [jethrokuan]
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username

.github/ISSUE_TEMPLATE/bug_report.md

@@ -39,5 +39,3 @@ Example:
 ### Environment
 
 <!-- Please M-x org-roam-diagnostics and paste results here -->
-
-- Org-roam commit: https://github.com/jethrokuan/org-roam/commit/commithashhere

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Install Eldev
-        run: curl -fsSL https://raw.github.com/doublep/eldev/master/webinstall/github-eldev | sh
+        run: curl -fsSL https://raw.github.com/org-roam/org-roam/master/github-eldev | sh
 
       - name: Install dependencies
         run: make prepare

CHANGELOG.md

@@ -1,13 +1,87 @@
 # Changelog
-
-## TBD
+## 2.2.2
+### Breaking
 ### Added
+- [#2138](https://github.com/org-roam/org-roam/pull/2138) export: add new module
+- [#2170](https://github.com/org-roam/org-roam/pull/2170) log: add new module for working with org logs
+- [#2158](https://github.com/org-roam/org-roam/pull/2158) db: support emacsql-sqlite-builtin and emacsql-sqlite-module
+- [#2160](https://github.com/org-roam/org-roam/pull/2160) core: support a list of `org-roam-file-exclude-regexp`
+
+### Removed
+### Fixed
+- [#2091](https://github.com/org-roam/org-roam/pull/2091) node: fix org-roam-promote-entire-buffer structural errors
+- [#2130](https://github.com/org-roam/org-roam/pull/2130) buffer: unlinked-references section now also searches within symlinked directories
+- [#2152](https://github.com/org-roam/org-roam/pull/2152) org-roam-preview-default-function: doesn't copy copy content of next heading node when current node's content is empty
+- [#2159](https://github.com/org-roam/org-roam/pull/2159) db: fix db syncs on narrowed buffers
+- [#2156](https://github.com/org-roam/org-roam/pull/2157) capture: templates with functions are handled correctly to avoid signaling `char-or-string-p`
+
+### Changed
+- [#2160](https://github.com/org-roam/org-roam/pull/2160) core: ignore files in `org-attach-id-dir` by default
+
+## 2.2.1
+### Breaking
+- [#2054](https://github.com/org-roam/org-roam/pull/2054) node: simplify default `org-roam-node-display-template`.
+  This was done so completions work fine by default on all completion systems. To restore the tabular vertical completion interface, set this in your configuration:
+
+  ```emacs-lisp
+  (setq org-roam-node-display-template
+        (concat "${title:*} "
+                (propertize "${tags:10}" 'face 'org-tag)))
+  ```
+
+### Added
+- [#2042](https://github.com/org-roam/org-roam/pull/2042) db: add `org-roam-db-extra-links-elements` and `org-roam-db-extra-links-exclude-keys` for fine-grained control over additional link parsing
+- [#2049](https://github.com/org-roam/org-roam/pull/2049) capture: allow ID to be used as part of `org-roam-capture-templates`
+- [#2050](https://github.com/org-roam/org-roam/pull/2050) core: add `FILTER-FN` to `org-roam-node-random`
+- [#2065](https://github.com/org-roam/org-roam/pull/2065) dailies: add `keys` argument to the remaining dailies functions `org-roam-dailies-goto-yesterday`/`-today`/`-tomorrow`/`-date` and `org-roam-dailies-capture-yesterday`/`-tomorrow`/`-date` to give the abilty to get into a capture buffer bypassing the selection screen in all dailies commands. Extension of #2055
+- [#2079](https://github.com/org-roam/org-roam/pull/2079) capture: ensure that `:ref` info captured in all cases.
+- [#2121](https://github.com/org-roam/org-roam/pull/2121) buffer: add unique option to `org-roam-backlinks-section`
+
+### Removed
+### Fixed
+- [#2086](https://github.com/org-roam/org-roam/pull/2086) capture: correctly update org-id-locations on capture
+- [#2082](https://github.com/org-roam/org-roam/pull/2082) buffer: don't destroy window if `org-roam-node-toggle` reuses window
+- [#2080](https://github.com/org-roam/org-roam/pull/2080) dailies: prevent multiple "dailies/" subdir expansions
+- [#2055](https://github.com/org-roam/org-roam/pull/2055) dailies: removed stray f require, which was causing require and compilation errors
+- [#2117](https://github.com/org-roam/org-roam/pull/2117) capture: preserve trailing whitespace content in capture templates
+
+### Changed
+- [#2060](https://github.com/org-roam/org-roam/pull/2060) node: added double acute accent normalization for Unicode characters in titles
+- [#2040](https://github.com/org-roam/org-roam/pull/2040) completions: fix completions display-width for Helm users
+- [#2025](https://github.com/org-roam/org-roam/pull/2025) chore: removed the dependencies on f.el and s.el
+- [#2109](https://github.com/org-roam/org-roam/pull/2109) capture: `org-roam-node-insert` places cursor after inserted link where appropriate
+- [#2123](https://github.com/org-roam/org-roam/pull/2123), [#2124](https://github.com/org-roam/org-roam/pull/2124) buffer: `org-roam-mode-section-functions` renamed to `org-roam-mode-sections`, supports passing args into the section-rendering function
+
+## 2.2.0
+### Added
+- [#1806](https://github.com/org-roam/org-roam/pull/1806), [#2017](https://github.com/org-roam/org-roam/pull/2017) db: support caching and usage of Org 9.5's in-built citations
+- [#1963](https://github.com/org-roam/org-roam/pull/1963) db: cache file title into files table
+- [#1977](https://github.com/org-roam/org-roam/pull/1977) db: support Org-ref v3 citations
+- [#1907](https://github.com/org-roam/org-roam/pull/1907), [#2009](https://github.com/org-roam/org-roam/pull/2009), [#2018](https://github.com/org-roam/org-roam/pull/2018) db: support sqlite3
+- [#2028](https://github.com/org-roam/org-roam/pull/2028) dailies: add `keys` argument to `org-roam-dailies-capture-today` and `org-roam-dailies--capture` functions to give the abilty to get into a capture buffer bypassing the selection screen
+
 ### Removed
 ### Changed
 - [#1795](https://github.com/org-roam/org-roam/pull/1795) buffer: optimized reflinks fetch
+- [#1809](https://github.com/org-roam/org-roam/pull/1809) capture: the mandatory `:if-new` property of each capture template is now renamed to `:target`
+- [#1829](https://github.com/org-roam/org-roam/pull/1829) perf: file sql updates are now wrapped in a transaction
+- [#1877](https://github.com/org-roam/org-roam/pull/1877) dailies: stop asking for time, only date
+- [#1949](https://github.com/org-roam/org-roam/pull/1949) db: check for property drawers are now case-insensitive
 
 ### Fixed
 - [#1798](https://github.com/org-roam/org-roam/pull/1798) org-roam-node-at-point: do not skip invisible headings
+- [#1807](https://github.com/org-roam/org-roam/pull/1807) capture: always trigger `:if-new` template for existing nodes
+- [#1813](https://github.com/org-roam/org-roam/pull/1813) db: prevent empty ROAM_ALIASES from crashing db updates
+- [#1816](https://github.com/org-roam/org-roam/pull/1816) db: prevent invalid ROAM_REFS from crashing db updates
+- [#1893](https://github.com/org-roam/org-roam/pull/1893), [#1896](https://github.com/org-roam/org-roam/pull/1896), [#1901](https://github.com/org-roam/org-roam/pull/1901), [#1895](https://github.com/org-roam/org-roam/pull/1895), [#1904](https://github.com/org-roam/org-roam/pull/1904) completions: various mini-buffer related completion fixes
+- [#1931](https://github.com/org-roam/org-roam/pull/1931) utils: org-roam-set-keyword now skips over all drawers
+- [#1947](https://github.com/org-roam/org-roam/pull/1947) db: links in ROAM_REFS are no longer considered as links
+- [#1948](https://github.com/org-roam/org-roam/pull/1948) completions: fix same-line completions
+- [#1953](https://github.com/org-roam/org-roam/pull/1953) db: refresh CATEGORY before writing to db
+- [#1946](https://github.com/org-roam/org-roam/pull/1946), [#1946](https://github.com/org-roam/org-roam/pull/1946), [#1958](https://github.com/org-roam/org-roam/pull/1958) various performance improvements
+- [#1980](https://github.com/org-roam/org-roam/pull/1980) utils: fix org-roam-with-file changing the minor-mode
+- [#2016](https://github.com/org-roam/org-roam/pull/2016) db: fix node caching being affected by agenda variables
+- [#2033](https://github.com/org-roam/org-roam/pull/2023) db: respect local variables during db parsing
 
 ## 2.1.0
 ### Added

README.md

@@ -39,7 +39,7 @@ detailed information, please read the [manual][docs].
 
 ### Using `package.el`
 <details>
-<summary>Toggle instuctions</summary>
+<summary>Toggle instructions</summary>
 
 You can install `org-roam` from [MELPA](https://melpa.org/) or [MELPA
 Stable](https://stable.melpa.org/) using `package.el`:
@@ -47,35 +47,11 @@ Stable](https://stable.melpa.org/) using `package.el`:
 ```
 M-x package-install RET org-roam RET
 ```
-
-Here's a very basic sample for configuration of `org-roam` using `use-package`:
-
-```emacs-lisp
-(use-package org-roam
-  :ensure t
-  :custom
-  (org-roam-directory (file-truename "/path/to/org-files/"))
-  :bind (("C-c n l" . org-roam-buffer-toggle)
-         ("C-c n f" . org-roam-node-find)
-         ("C-c n g" . org-roam-graph)
-         ("C-c n i" . org-roam-node-insert)
-         ("C-c n c" . org-roam-capture)
-         ;; Dailies
-         ("C-c n j" . org-roam-dailies-capture-today))
-  :config
-  (org-roam-db-autosync-mode)
-  ;; If using org-roam-protocol
-  (require 'org-roam-protocol))
-```
-
-Note that the `file-truename` function is only necessary when you use symbolic
-link to `org-roam-directory`. Org-roam won't automatically resolve symbolic link
-to the directory.
 </details>
 
 ### Using `straight.el`
 <details>
-<summary>Toggle instuctions</summary>
+<summary>Toggle instructions</summary>
 
 Installation from MELPA or MELPA Stable using `straight.el`:
 
@@ -115,7 +91,7 @@ next sample will get you there:
 
 ### Using Doom Emacs
 <details>
-<summary>Toggle instuctions</summary>
+<summary>Toggle instructions</summary>
 
 Doom's `:lang org` module comes with support for `org-roam`, but it's not
 enabled by default. To activate it pass `+roam2` flag to `org` module in your
@@ -198,6 +174,33 @@ Org-roam also comes with `.texi` files to integrate with Emacs' built-in Info
 system. Read the manual to find more details for how to install them manually.
 </details>
 
+## Configuration
+
+Here's a very basic sample for configuration of `org-roam` using `use-package`:
+
+```emacs-lisp
+(use-package org-roam
+  :ensure t
+  :custom
+  (org-roam-directory (file-truename "/path/to/org-files/"))
+  :bind (("C-c n l" . org-roam-buffer-toggle)
+         ("C-c n f" . org-roam-node-find)
+         ("C-c n g" . org-roam-graph)
+         ("C-c n i" . org-roam-node-insert)
+         ("C-c n c" . org-roam-capture)
+         ;; Dailies
+         ("C-c n j" . org-roam-dailies-capture-today))
+  :config
+  ;; If you're using a vertical completion framework, you might want a more informative completion interface
+  (setq org-roam-node-display-template (concat "${title:*} " (propertize "${tags:10}" 'face 'org-tag)))
+  (org-roam-db-autosync-mode)
+  ;; If using org-roam-protocol
+  (require 'org-roam-protocol))
+```
+
+Note that the `file-truename` function is only necessary when you use symbolic
+link to `org-roam-directory`. Org-roam won't automatically resolve symbolic link
+to the directory.
 ## Getting Started
 
 [David Wilson](https://github.com/daviwil) of [System
@@ -225,7 +228,7 @@ it has not already been addressed on [GitHub][issues] or on
 
 - [Jethro Kuan](https://braindump.jethro.dev/)
   ([Source](https://github.com/jethrokuan/braindump/tree/master/org))
-- [Alexey Shmalko](https://braindump.rasen.dev/)
+- [Alexey Shmalko](https://www.alexeyshmalko.com/)
 - [Sidharth Arya](https://sidhartharya.github.io/braindump/index.html)
 
 ## Contributing
@@ -249,6 +252,6 @@ General Public License, Version 3.
 [release]: https://github.com/org-roam/org-roam/releases
 [docs]: https://www.orgroam.com/manual.html
 [discourse]: https://org-roam.discourse.group/
-[slack]: https://join.slack.com/t/orgroam/shared_invite/zt-deoqamys-043YQ~s5Tay3iJ5QRI~Lxg
+[slack]: https://join.slack.com/t/orgroam/shared_invite/zt-wuoize1z-x3UyQnQ0WHF0RhuEQ2NLnQ
 [issues]: https://github.com/org-roam/org-roam/issues
 [faq]: https://www.orgroam.com/manual.html#FAQ

doc/index.html

@@ -95,7 +95,7 @@
         <ul>
             <li>Read our documentation within Emacs, or on the <a href="https://www.orgroam.com/manual.html">Online Manual</a></li>
             <li>Participate in our forum discussions on <a href="https://org-roam.discourse.group">Discourse</a></li>
-            <li>Chat with us on <a href="https://join.slack.com/t/orgroam/shared_invite/zt-deoqamys-043YQ~s5Tay3iJ5QRI~Lxg">Slack</a></li>
+            <li>Chat with us on <a href="https://join.slack.com/t/orgroam/shared_invite/zt-wuoize1z-x3UyQnQ0WHF0RhuEQ2NLnQ">Slack</a></li>
           </ul>
       </div>
     </section>
@@ -151,7 +151,7 @@
         <div class="row">
           <a
             class="content footer-links"
-            href="https://join.slack.com/t/orgroam/shared_invite/zt-deoqamys-043YQ~s5Tay3iJ5QRI~Lxg"
+            href="https://join.slack.com/t/orgroam/shared_invite/zt-wuoize1z-x3UyQnQ0WHF0RhuEQ2NLnQ"
             >Slack</a
           >
         </div>

doc/org-roam.org

@@ -1,23 +1,23 @@
 #+title: Org-roam User Manual
 #+author: Jethro Kuan
 #+email: jethrokuan95@gmail.com
-#+date: 2020-2021
+#+date: 2020-2022
 #+language: en
 
 #+texinfo_deffn: t
 #+texinfo_dir_category: Emacs
 #+texinfo_dir_title: Org-roam: (org-roam).
 #+texinfo_dir_desc: Roam Research for Emacs.
-#+subtitle: for version 2.1.0
+#+subtitle: for version 2.2.1
 
 #+options: H:4 num:3 toc:nil creator:t ':t
 #+property: header-args :eval never
 #+texinfo: @noindent
 
-This manual is for Org-roam version 2.1.0.
+This manual is for Org-roam version 2.2.1.
 
 #+BEGIN_QUOTE
-Copyright (C) 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+Copyright (C) 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 You can redistribute this document and/or modify it under the terms of the GNU
 General Public License as published by the Free Software Foundation, either
@@ -134,7 +134,7 @@ A slip-box requires a method for quickly capturing ideas. These are called
 *fleeting notes*: they are simple reminders of information or ideas that will
 need to be processed later on, or trashed. This is typically accomplished using
 ~org-capture~ (see info:org#Capture), or using Org-roam's daily notes
-functionality (see [[*Org-roam Dailies][Org-roam Dailies]]). This provides a central inbox for collecting
+functionality (see [[*org-roam-dailies][org-roam-dailies]]). This provides a central inbox for collecting
 thoughts, to be processed later into permanent notes.
 
 *Permanent notes*
@@ -288,29 +288,25 @@ in your Emacs environment as a prerequisite for Org-roam when you install it.
 ~emacsql-sqlite~ requires a C compiler (e.g. ~gcc~ or ~clang~) to be present in
 your computer. How to install a C compiler depends on the OS that you use.
 
-- For Windows:
+**** C Compiler for Windows
 
-There are various ways to install one, depending on how you have installed
-Emacs. If you use Emacs within a Cygwin or MinGW environment, then you should
-install a compiler using their respective package manager.
+One of the easiest ways to install a C compiler in Windows is to use [[https://www.msys2.org/][MSYS2]] as at the time of this writing:
 
-If you have installed your Emacs from the [[https://www.gnu.org/software/emacs/][GNU Emacs website]], then the easiest way
-is to use [[https://www.msys2.org/][MSYS2]] as at the time of this writing:
-
-1. Use the installer in the official website and install MSYS2
-2. Run MSYS2
-3. In the command-line tool, type the following and answer "Y" to proceed:
+1. Download and use the installer in the official MSYS2 website
+2. Run MSYS2 and in its terminal, type the following and answer "Y" to
+   proceed -- this will install ~gcc~ in your PC:
 
    #+BEGIN_SRC bash
      pacman -S gcc
    #+END_SRC
 
-   Note that you do not need to manually set the PATH for MSYS2; the
-installer automatically takes care of it for you.
+4. On Windows, add ~C:\msys64\usr\bin~ (command =where gcc= in MSYS2 terminal
+   can tell you the correct path) to ~PATH~ in your environmental variables
 
-4. Open Emacs and call ~M-x org-roam-db-autosync-mode~
+5. Launch Emacs and call ~M-x org-roam-db-autosync-mode~ (launch Emacs after
+   defining the path, so that Emacs can recognize it)
 
-   This will automatically start compiling ~emacsql-sqlite~; you should see a
+This will automatically start compiling ~emacsql-sqlite~; you should see a
 message in minibuffer. It may take a while until compilation completes. Once
 complete, you should see a new file ~emacsql-sqlite.exe~ created in a subfolder
 named ~sqlite~ under ~emacsql-sqlite~ installation folder. It's typically in
@@ -373,7 +369,12 @@ For this tutorial, create an empty directory, and set ~org-roam-directory~:
 #+END_SRC
 
 The ~file-truename~ function is only necessary when you use symbolic links
-inside ~org-roam-directory~: Org-roam does not resolve symbolic links.
+inside ~org-roam-directory~: Org-roam does not resolve symbolic links. One can
+however instruct Emacs to always resolve symlinks, at a performance cost:
+
+#+begin_src emacs-lisp
+  (setq find-file-visit-truename t)
+#+end_src
 
 Next, we setup Org-roam to run functions on file changes to maintain cache
 consistency. This is achieved by running ~M-x org-roam-db-autosync-mode~. To
@@ -417,7 +418,100 @@ brought through the node creation process.
 One can also conveniently insert links via the completion-at-point functions
 Org-roam provides (see [[*Completion][Completion]]).
 
+** Customizing Node Completions
+
+Node selection is achieved via the ~completing-read~ interface, typically
+through `org-roam-node-read`. The presentation of these nodes are governed by
+~org-roam-node-display-template~.
+
+- Variable: org-roam-node-display-template
+
+  Configures display formatting for Org-roam node.
+
+  Patterns of form "${field-name:length}" are interpolated based
+  on the current node.
+
+  Each "field-name" is replaced with the return value of each
+  corresponding accessor function for org-roam-node, e.g.
+  "${title}" will be interpolated by the result of
+  org-roam-node-title. You can also define custom accessors using
+  cl-defmethod. For example, you can define:
+
+    (cl-defmethod org-roam-node-my-title ((node org-roam-node))
+      (concat "My " (org-roam-node-title node)))
+
+  and then reference it here or in the capture templates as
+  "${my-title}".
+
+  "length" is an optional specifier and declares how many
+  characters can be used to display the value of the corresponding
+  field. If it's not specified, the field will be inserted as is,
+  i.e. it won't be aligned nor trimmed. If it's an integer, the
+  field will be aligned accordingly and all the exceeding
+  characters will be trimmed out. If it's "*", the field will use
+  as many characters as possible and will be aligned accordingly.
+
+  A closure can also be assigned to this variable in which case the
+  closure is evaluated and the return value is used as the
+  template. The closure must evaluate to a valid template string.
+
+If you're using a vertical completion framework, such as Ivy and Selectrum,
+Org-roam supports the generation of an aligned, tabular completion interface.
+For example, to include a column for tags up to 10 character widths wide, one
+can set ~org-roam-node-display-template~ as such:
+
+#+begin_src emacs-lisp
+  (setq org-roam-node-display-template
+        (concat "${title:*} "
+                (propertize "${tags:10}" 'face 'org-tag)))
+#+end_src
+
 * Customizing Node Caching
+** How to cache
+
+Org-roam uses a sqlite database to perform caching, but there are multiple Emacs
+libraries that can be used. The default used by Org-roam is ~emacs-sqlite~.
+Below the pros and cons of each package is used:
+
+[[https://github.com/skeeto/emacsql][**emacs-sqlite**]]
+
+The default option used by Org-roam. This library is the most mature and
+well-supported and is imported by default in Org-roam.
+
+One downside of using ~emacs-sqlite~ is that using it requires compilation and
+can cause issues in some environments (especially Windows). If you have issues
+producing the customized binary required by ~emacs-sqlite~, consider using
+~emacs-sqlite3~.
+
+[[https://github.com/cireu/emacsql-sqlite3][**emacs-sqlite3**]]
+
+~emacs-sqlite3~ uses the official sqlite3 binary that can be obtained from your
+system's package manager. This is useful if you have issues producing the
+~sqlite3~ binary required by the other packages. However, it is not recommended
+because it has some compatibility issues with Emacs, but should work for most
+regular cases. See [[https://nullprogram.com/blog/2014/02/06/][Chris Wellon's blog post]] for more information.
+
+To use ~emacsql-sqlite3~, ensure that the package is installed, and set:
+
+#+begin_src emacs-lisp
+  (setq org-roam-database-connector 'sqlite3)
+#+end_src
+
+[[https://github.com/emacscollective/emacsql-libsqlite3/][**emacsql-libsqlite3**]]
+
+~emacs-libsqlite3~ is a relatively young package which uses an Emacs module that
+exposes parts of the SQLite C API to Emacs Lisp, instead of using subprocess as
+~emacsql-sqlite~ does. It is expected to be a more performant drop-in
+replacement for ~emacs-sqlite~.
+
+At the moment it is experimental and does not work well with the SQL query load
+required by Org-roam, but you may still try it by ensuring the package is
+installed and setting:
+
+#+begin_src emacs-lisp
+  (setq org-roam-database-connector 'libsqlite3)
+#+end_src
+
 ** What to cache
 
 By default, all nodes (any headline or file with an ID) are cached by Org-roam.
@@ -444,6 +538,35 @@ all headlines with the ~ATTACH~ tag from the Org-roam database, one can set:
         (not (member "ATTACH" (org-get-tags)))))
 #+end_src
 
+Org-roam relied on the obtained Org AST for the buffer to parse links. However,
+links appearing in some places (e.g. within property drawers) are not considered
+by the Org AST to be links. Therefore, Org-roam takes special care of
+additionally trying to process these links. Use
+~org-roam-db-extra-links-elements~ to specify which additional Org AST element
+types to consider.
+
+- Variable: org-roam-db-extra-links-elements
+
+  The list of Org element types to include for parsing by Org-roam.
+
+  By default, when parsing Org's AST, links within keywords and
+  property drawers are not parsed as links. Sometimes however, it
+  is desirable to parse and cache these links (e.g. hiding links in
+  a property drawer).
+
+Additionally, one may want to ignore certain keys from being excluded within
+property drawers. For example, we would not want ~ROAM_REFS~ links to be
+self-referential. Hence, to exclude specific keys, we use
+~org-roam-db-extra-links-exclude-keys~.
+
+- Variable: org-roam-db-extra-links-exclude-keys
+
+  Keys to ignore when mapping over links.
+
+  The car of the association list is the Org element type (e.g. keyword). The
+  cdr is a list of case-insensitive strings to exclude from being treated as
+  links.
+
 ** When to cache
 
 By default, Org-roam is eager in caching: each time an Org-roam file is modified
@@ -495,7 +618,7 @@ keybindings available. Here are several of the more useful ones:
 
 - ~M-{N}~: ~magit-section-show-level-{N}-all~
 - ~n~: ~magit-section-forward~
--~<TAB>~: ~magit-section-toggle~
+- ~<TAB>~: ~magit-section-toggle~
 - ~<RET>~: ~org-roam-buffer-visit-thing~
 
 ~org-roam-buffer-visit-thing~ is a placeholder command, that is replaced by
@@ -510,10 +633,10 @@ There are currently 3 provided widget types:
 - Unlinked references :: View nodes that contain text that match the nodes
   title/alias but are not linked
 
-To configure what sections are displayed in the buffer, set ~org-roam-mode-section-functions~.
+To configure what sections are displayed in the buffer, set ~org-roam-mode-sections.
 
 #+begin_src emacs-lisp
-  (setq org-roam-mode-section-functions
+  (setq org-roam-mode-sections
         (list #'org-roam-backlinks-section
               #'org-roam-reflinks-section
               ;; #'org-roam-unlinked-references-section
@@ -522,6 +645,16 @@ To configure what sections are displayed in the buffer, set ~org-roam-mode-secti
 
 Note that computing unlinked references may be slow, and has not been added in by default.
 
+For each section function, you can pass args along to modify its behaviour. For
+example, if you want to render unique sources for backlinks (and also keep
+rendering reference links), set ~org-roam-mode-sections~ as follows:
+
+#+begin_src emacs-lisp
+  (setq org-roam-mode-sections
+        '((org-roam-backlinks-section :unique t)
+          org-roam-reflinks-section))
+#+end_src
+
 ** Configuring the Org-roam buffer display
 
 Org-roam does not control how the pop-up buffer is displayed: this is left to
@@ -629,7 +762,7 @@ With the above example, if another node links to https://www.google.com/, it
 will show up as a “reference backlink”.
 
 These keys also come in useful for when taking website notes, using the
- ~roam-ref~ protocol (see [[*Org-roam Protocol][Roam Protocol]]).
+ ~roam-ref~ protocol (see [[*org-roam-protocol][org-roam-protocol]]).
 
 You may assign multiple refs to a single node, for example when you want
 multiple papers in a series to share the same note, or an article has a citation
@@ -646,6 +779,59 @@ Org-roam also provides some functions to add or remove refs.
 
   Remove a ref from the node at point.
 
+* Citations
+
+Since version 9.5, Org has first-class support for citations. Org-roam supports
+the caching of both these in-built citations (of form ~[cite:@key]~) and [[https://github.com/jkitchin/org-ref][org-ref]]
+citations (of form cite:key).
+
+Org-roam attempts to load both the ~org-ref~ and ~org-cite~ package when
+indexing files, so no further setup from the user is required for citation
+support.
+
+** Using the Cached Information
+
+It is common to use take reference notes for academic papers. To designate the
+node to be the canonical node for the academic paper, we can use its unique
+citation key:
+
+#+begin_src org
+,* Probabilistic Robotics
+:PROPERTIES:
+:ID:       51b7b82c-bbb4-4822-875a-ed548cffda10
+:ROAM_REFS: @thrun2005probabilistic
+:END:
+#+end_src
+
+or
+
+#+begin_src org
+,* Probabilistic Robotics
+:PROPERTIES:
+:ID:       51b7b82c-bbb4-4822-875a-ed548cffda10
+:ROAM_REFS: [cite:@thrun2005probabilistic]
+:END:
+#+end_src
+
+for ~org-cite~, or:
+
+#+begin_src org
+,* Probabilistic Robotics
+:PROPERTIES:
+:ID:       51b7b82c-bbb4-4822-875a-ed548cffda10
+:ROAM_REFS: cite:thrun2005probabilistic
+:END:
+#+end_src
+
+for ~org-ref~.
+
+When another node has a citation for that key, we can see it using the
+~Reflinks~ section of the Org-roam buffer.
+
+Extension developers may be interested in retrieving the citations within their
+notes. This information can be found within the ~citation~ table of the Org-roam
+database.
+
 * Completion
 
 Completions for Org-roam are provided via ~completion-at-point~. Org-roam
@@ -698,7 +884,7 @@ extension in your Org-roam capture templates. For example:
 
 #+begin_src emacs-lisp
 (setq org-roam-capture-templates '(("d" "default" plain "%?"
-     :if-new (file+head "${slug}.org.gpg"
+     :target (file+head "${slug}.org.gpg"
                         "#+title: ${title}\n")
      :unnarrowed t)))
 #+end_src
@@ -707,13 +893,95 @@ Note that the Org-roam database stores metadata information in plain-text
 (headline text, for example), so if this information is private to you then you
 should also ensure the database is encrypted.
 
-* Org-roam Protocol
+* The Templating System
+
+Org-roam extends the ~org-capture~ system, providing a smoother note-taking
+experience. However, these extensions mean Org-roam capture templates are
+incompatible with ~org-capture~ templates.
+
+Org-roam's templates are specified by ~org-roam-capture-templates~. Just like
+~org-capture-templates~, ~org-roam-capture-templates~ can contain multiple
+templates. If ~org-roam-capture-templates~ only contains one template, there
+will be no prompt for template selection.
+
+** Template Walkthrough
+
+To demonstrate the additions made to org-capture templates. Here, we explain
+the default template, reproduced below. You will find most of the elements
+of the template are similar to ~org-capture~ templates.
+
+#+BEGIN_SRC emacs-lisp
+(("d" "default" plain "%?"
+  :target (file+head "%<%Y%m%d%H%M%S>-${slug}.org"
+                     "#+title: ${title}\n")
+  :unnarrowed t))
+#+END_SRC
+
+1. The template has short key ~"d"~. If you have only one template, org-roam
+   automatically chooses this template for you.
+2. The template is given a description of ~"default"~.
+3. ~plain~ text is inserted. Other options include Org headings via
+   ~entry~.
+4. Notice that the ~target~ that's usually in Org-capture templates is missing
+   here.
+5. ~"%?"~ is the template inserted on each call to ~org-roam-capture-~.
+   This template means don't insert any content, but place the cursor here.
+6. ~:target~ is a compulsory specification in the Org-roam capture template. The
+   first element of the list indicates the type of the target, the second
+   element indicates the location of the captured node, and the rest of the
+   elements indicate prefilled template that will be inserted and the position
+   of the point will be adjusted for. The latter behavior varies from type to
+   type of the capture target.
+7. ~:unnarrowed t~ tells org-capture to show the contents for the whole file,
+   rather than narrowing to just the entry. This is part of the Org-capture
+   templates.
+
+See the ~org-roam-capture-templates~ documentation for more details and
+customization options.
+
+** Org-roam Template Expansion
+
+Org-roam's template definitions also extend org-capture's template syntax, to
+allow prefilling of strings. We have seen a glimpse of this in [[*Template Walkthrough][Template
+Walkthrough]].
+
+Org-roam provides the ~${foo}~ syntax for substituting variables with known
+strings. ~${foo}~'s substitution is performed as follows:
+
+1. If ~foo~ is a function, ~foo~ is called with the current node as its
+   argument.
+2. Else if ~org-roam-node-foo~ is a function, ~foo~ is called with the current node
+   as its argument. The ~org-roam-node-~ prefix defines many of Org-roam's node
+   accessors such as ~org-roam-node-title~ and ~org-roam-node-level~.
+3. Else look up ~org-roam-capture--info~ for ~foo~. This is an internal variable
+   that is set before the capture process begins.
+4. If none of the above applies, read a string using ~completing-read~.
+   a. Org-roam also provides the ~${foo=default_val}~ syntax, where if a default
+      value is provided, will be the initial value for the ~foo~ key during
+      minibuffer completion.
+
+One can check the list of available keys for nodes by inspecting the
+~org-roam-node~ struct. At the time of writing, it is:
+
+#+begin_src emacs-lisp
+  (cl-defstruct (org-roam-node (:constructor org-roam-node-create)
+                               (:copier nil))
+    "A heading or top level file with an assigned ID property."
+    file file-hash file-atime file-mtime
+    id level point todo priority scheduled deadline title properties olp
+    tags aliases refs)
+#+end_src
+
+This makes ~${file}~, ~${file-hash}~ etc. all valid substitutions.
+
+* Extensions
+** org-roam-protocol
 
 Org-roam provides extensions for capturing content from external applications
 such as the browser, via ~org-protocol~. Org-roam extends ~org-protocol~ with 2
 protocols: the ~roam-node~ and ~roam-ref~ protocols.
 
-** Installation
+*** Installation
 
 To enable Org-roam's protocol extensions, simply add the following to your init
 file:
@@ -723,9 +991,18 @@ file:
 #+END_SRC
 
 We also need to set up ~org-protocol~: the instructions for setting up
-~org-protocol~ are reproduced below.
+~org-protocol~ are reproduced here.
 
-*** Linux
+On a high-level, external calls are passed to Emacs via ~emacsclient~.
+~org-protocol~ intercepts these and runs custom actions based on the protocols
+registered. Hence, to use ~org-protocol~, once must:
+
+1. launch the ~emacsclient~ process
+2. Register ~org-protocol://~ as a valid scheme-handler
+
+The instructions for the latter for each operating system is detailed below.
+
+**** Linux
 For Linux users, create a desktop application in
 ~~/.local/share/applications/org-protocol.desktop~:
 
@@ -766,7 +1043,7 @@ make the new policy take effect.
 See [[https://www.chromium.org/administrators/linux-quick-start][here]] for more info on the ~/etc/opt/chrome/policies/managed~ directory and
 [[https://cloud.google.com/docs/chrome-enterprise/policies/?policy=ExternalProtocolDialogShowAlwaysOpenCheckbox][here]] for information on the ~ExternalProtocolDialogShowAlwaysOpenCheckbox~ policy.
 
-*** Mac OS
+**** Mac OS
 For Mac OS, we need to create our own application.
 
 1. Launch Script Editor
@@ -821,7 +1098,30 @@ defaults write com.apple.LaunchServices/com.apple.launchservices.secure LSHandle
 
 Then restart your computer.
 
-*** Windows
+***** Testing org-protocol
+
+To test that you have the handler setup and registered properly from the command
+line you can run:
+
+#+begin_src bash
+  open org-protocol://roam-ref\?template=r\&ref=test\&title=this
+#+end_src
+
+If you get an error similar too this or the wrong handler is run:
+
+#+begin_quote
+No application knows how to open URL org-protocol://roam-ref?template=r&ref=test&title=this (Error Domain=NSOSStatusErrorDomain Code=-10814 "kLSApplicationNotFoundErr: E.g. no application claims the file" UserInfo={_LSLine=1489, _LSFunction=runEvaluator}).
+
+#+end_quote
+
+You may need to manually register your handler, like this:
+
+#+begin_src bash
+/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/Support/lsregister -R  -f /Applications/OrgProtocolClient.app
+#+end_src
+
+Here is a link to the lsregister command that is really useful: https://eclecticlight.co/2019/03/25/lsregister-a-valuable-undocumented-command-for-launchservices/
+**** Windows
 For Windows, create a temporary ~org-protocol.reg~ file:
 
 #+BEGIN_SRC text
@@ -846,13 +1146,13 @@ Windows, replace the last line with:
 After executing the .reg file, the protocol is registered and you can delete the
 file.
 
-** The roam-node protocol
+*** The roam-node protocol
 
 The roam-node protocol opens the node with ID specified by the ~node~ key (e.g.
 ~org-protocol://roam-node?node=node-id~). ~org-roam-graph~ uses this to make the
 graph navigable.
 
-** The roam-ref protocol
+*** The roam-ref protocol
 
 This protocol finds or creates a new note with a given ~ROAM_REFS~:
 
@@ -880,89 +1180,11 @@ or as a keybinding in ~qutebrowser~ in , using the ~config.py~ file (see
 where ~template~ is the template key for a template in
 ~org-roam-capture-ref-templates~ (see [[*The Templating System][The Templating System]]).
 
-* The Templating System
-
-Org-roam extends the ~org-capture~ system, providing a smoother note-taking
-experience. However, these extensions mean Org-roam capture templates are
-incompatible with ~org-capture~ templates.
-
-Org-roam's templates are specified by ~org-roam-capture-templates~. Just like
-~org-capture-templates~, ~org-roam-capture-templates~ can contain multiple
-templates. If ~org-roam-capture-templates~ only contains one template, there
-will be no prompt for template selection.
-
-** Template Walkthrough
-
-To demonstrate the additions made to org-capture templates. Here, we explain
-the default template, reproduced below. You will find some most of the elements
-of the template are similar to ~org-capture~ templates.
-
-#+BEGIN_SRC emacs-lisp
-(("d" "default" plain "%?"
-  :if-new (file+head "%<%Y%m%d%H%M%S>-${slug}.org"
-                     "#+title: ${title}\n")
-  :unnarrowed t))
-#+END_SRC
-
-1. The template has short key ~"d"~. If you have only one template, org-roam
-   automatically chooses this template for you.
-2. The template is given a description of ~"default"~.
-3. ~plain~ text is inserted. Other options include Org headings via
-   ~entry~.
-4. Notice that the ~target~ that's usually in Org-capture templates is missing
-   here.
-5. ~"%?"~ is the template inserted on each call to ~org-roam-capture-~.
-   This template means don't insert any content, but place the cursor here.
-6. ~:if-new~ is a compulsory specification in the Org-roam capture template.
-   This indicates the location for the new node.
-7. ~:unnarrowed t~ tells org-capture to show the contents for the whole file,
-   rather than narrowing to just the entry. This is part of the Org-capture
-   templates.
-
-See the ~org-roam-capture-templates~ documentation for more details and
-customization options.
-
-** Org-roam Template Expansion
-
-Org-roam's template definitions also extend org-capture's template syntax, to
-allow prefilling of strings. We have seen a glimpse of this in [[*Template Walkthrough][Template
-Walkthrough]].
-
-Org-roam provides the ~${foo}~ syntax for substituting variables with known
-strings. ~${foo}~'s substitution is performed as follows:
-
-1. If ~foo~ is a function, ~foo~ is called with the current node as its
-   argument.
-2. Else if ~org-roam-node-foo~ is a function, ~foo~ is called with the current node
-   as its argument. The ~org-roam-node-~ prefix defines many of Org-roam's node
-   accessors such as ~org-roam-node-title~ and ~org-roam-node-level~.
-3. Else look up ~org-roam-capture--info~ for ~foo~. This is an internal variable
-   that is set before the capture process begins.
-4. If none of the above applies, read a string using ~completing-read~.
-   a. Org-roam also provides the ~${foo=default_val}~ syntax, where if a default
-      value is provided, will be the initial value for the ~foo~ key during
-      minibuffer completion.
-
-One can check the list of available keys for nodes by inspecting the
-~org-roam-node~ struct. At the time of writing, it is:
-
-#+begin_src emacs-lisp
-  (cl-defstruct (org-roam-node (:constructor org-roam-node-create)
-                               (:copier nil))
-    "A heading or top level file with an assigned ID property."
-    file file-hash file-atime file-mtime
-    id level point todo priority scheduled deadline title properties olp
-    tags aliases refs)
-#+end_src
-
-This makes ~${file}~, ~${file-hash}~ etc. all valid substitutions.
-
-* Graphing
+** org-roam-graph
 
 Org-roam provides basic graphing capabilities to explore interconnections
 between notes, in ~org-roam-graph~. This is done by performing SQL queries and
-generating images using [[https://graphviz.org/][Graphviz]]. The graph can also be navigated: see [[*Org-roam Protocol][Roam
-Protocol]].
+generating images using [[https://graphviz.org/][Graphviz]]. The graph can also be navigated: see [[*org-roam-protocol][org-roam-protocol]].
 
 The entry point to graph creation is ~org-roam-graph~.
 
@@ -1003,7 +1225,7 @@ ARG may be any of the following values:
           (org-roam-graph--open (concat "file://///wsl$/Ubuntu" file)))))
   #+END_SRC
 
-** Graph Options
+*** Graph Options
 
 Graphviz provides many options for customizing the graph output, and Org-roam
 supports some of them. See https://graphviz.gitlab.io/_pages/doc/info/attrs.html
@@ -1029,12 +1251,12 @@ for customizable options.
   Extra options for edges in the graphviz output (The "E" attributes).
   Example: ~'(("dir" . "back"))~
 
-* Org-roam Dailies
+** org-roam-dailies
 
 Org-roam provides journaling capabilities akin to
 Org-journal with ~org-roam-dailies~.
 
-** Configuration
+*** Configuration
 
 For ~org-roam-dailies~ to work, you need to define two variables:
 
@@ -1054,13 +1276,13 @@ Here is a sane default configuration:
 (setq org-roam-dailies-capture-templates
       '(("d" "default" entry
          "* %?"
-         :if-new (file+head "%<%Y-%m-%d>.org"
+         :target (file+head "%<%Y-%m-%d>.org"
                             "#+title: %<%Y-%m-%d>\n"))))
 #+end_src
 
 See [[*The Templating System][The Templating System]] for creating new templates.
 
-** Usage
+*** Usage
 
 ~org-roam-dailies~ provides these interactive functions:
 
@@ -1114,6 +1336,18 @@ There are also commands which allow you to use Emacs’s ~calendar~ to find the
 - Function: ~org-roam-dailies-goto-next-note~
 
   When in an daily-note, find the next one.
+** org-roam-export
+
+Because Org-roam files are plain org files, they can be exported easily using
+~org-export~ to a variety of formats, including ~html~ and ~pdf~. However,
+Org-roam relies heavily on ID links, which Org's html export has poor support
+of. To fix this, Org-roam provides a bunch of overrides to better support
+export. To use them, simply run:
+
+#+begin_src emacs-lisp
+  (require 'org-roam-export)
+#+end_src
+
 * Performance Optimization
 ** Garbage Collection
 
@@ -1187,7 +1421,7 @@ The Deft interface can slow down quickly when the number of files get huge.
 
 [[https://github.com/bastibe/org-journal][Org-journal]] provides journaling capabilities to Org-mode. A lot of its
 functionalities have been incorporated into Org-roam under the name
-[[*Org-roam Dailies][~org-roam-dailies~]]. It remains a good tool if you want to isolate your verbose
+[[*org-roam-dailies][~org-roam-dailies~]]. It remains a good tool if you want to isolate your verbose
 journal entries from the ideas you would write on a scratchpad.
 
 #+BEGIN_SRC emacs-lisp
@@ -1240,6 +1474,9 @@ documents (PDF, EPUB etc.) within Org-mode.
 
 ** Bibliography
 
+Org 9.5 added native citation and bibliography functionality, called "org-cite",
+which org-roam supports.
+
 [[https://github.com/org-roam/org-roam-bibtex][org-roam-bibtex]] offers tight integration between [[https://github.com/jkitchin/org-ref][org-ref]], [[https://github.com/tmalsburg/helm-bibtex][helm-bibtex]] and
 ~org-roam~. This helps you manage your bibliographic notes under ~org-roam~.
 
@@ -1267,20 +1504,27 @@ variable using directory-local variables. This is what ~.dir-locals.el~ may
 contain:
 
 #+BEGIN_SRC emacs-lisp
-  ((nil . ((org-roam-directory . (expand-file-name "."))
-           (org-roam-db-location . (expand-file-name "./org-roam.db")))))
+  ((nil . ((org-roam-directory . "/path/to/alt/org-roam-dir")
+           (org-roam-db-location . "/path/to/alt/org-roam-dir/org-roam.db"))))
+#+END_SRC
+
+Note ~org-roam-directory~ and ~org-roam-db-location~ should be an absolute path, not relative.
+
+Alternatively, use ~eval~ if you wish to call functions:
+
+#+BEGIN_SRC emacs-lisp
+  ((nil . ((eval . (setq-local
+                    org-roam-directory (expand-file-name (locate-dominating-file
+                                                          default-directory ".dir-locals.el"))))
+           (eval . (setq-local
+                    org-roam-db-location (expand-file-name "org-roam.db"
+                                                           org-roam-directory))))))
 #+END_SRC
 
 All files within that directory will be treated as their own separate set of
 Org-roam files. Remember to run ~org-roam-db-sync~ from a file within
 that directory, at least once.
 
-** How do I migrate from Roam Research?
-
-Fabio has produced a command-line tool that converts markdown files exported
-from Roam Research into Org-roam compatible markdown. More instructions are
-provided [[https://github.com/fabioberger/roam-migration][in the repository]].
-
 ** How do I create a note whose title already matches one of the candidates?
 
 This situation arises when, for example, one would like to create a note titled
@@ -1301,28 +1545,125 @@ you don't want them to be (e.g. when tangling an Org file), check the value you
 have set for ~org-id-link-to-org-use-id~: setting it to ~'create-if-interactive~
 is a popular option.
 
-* Migrating from Org-roam v1
+** How do I migrate from Roam Research?
+
+Fabio has produced a command-line tool that converts markdown files exported
+from Roam Research into Org-roam compatible markdown. More instructions are
+provided [[https://github.com/fabioberger/roam-migration][in the repository]].
+
+** How to migrate from Org-roam v1?
 
 Those coming from Org-roam v1 will do well treating v2 as entirely new software.
 V2 has a smaller core and fewer moving parts, while retaining the bulk of its
 functionality. It is recommended to read the documentation above about nodes.
 
-It is still desirable to migrate notes collected in v1 to v2. To migrate your v1
-notes to v2, you may use the migration script provided in [[https://gist.github.com/jethrokuan/02f41028fb4a6f81787dc420fb99b6e4][this gist]], or [[https://gist.github.com/jethrokuan/02f41028fb4a6f81787dc420fb99b6e4#gistcomment-3737019][this
-gist]], the latter being better tested. [[https://d12frosted.io/posts/2021-06-11-path-to-org-roam-v2.html][This blog post]] provides a good overview of
-what's new in v2 and how to migrate.
+It is still desirable to migrate notes collected in v1 to v2.
+To migrate your v1 notes to v2, use =M-x org-roam-migrate-wizard=.
+[[https://d12frosted.io/posts/2021-06-11-path-to-org-roam-v2.html][This blog post]]
+provides a good overview of what's new in v2 and how to migrate.
 
-Simply put, to migrate notes from v1 to v2, one must:
+Essentially, to migrate notes from v1 to v2, one must:
 
-1. Add IDs to all existing notes. These are located in top-level property
-   drawers (Although note that in v2, not all files need to have IDs)
+1. Add IDs to all existing notes.
+   These are located in top-level property drawers
+   (Although note that in v2, not all files need to have IDs).
 2. Update the Org-roam database to conform to the new schema.
 3. Replace ~#+ROAM_KEY~ into the ~ROAM_REFS~ property
 4. Replace ~#+ROAM_ALIAS~ into the ~ROAM_ALIASES~ property
-5. Move ~#+ROAM_TAGS~ into the ~#+FILETAGS~ property for file-level nodes, and
-   the ~ROAM_TAGS~ property for headline nodes
+5. Move ~#+ROAM_TAGS~ into the ~#+FILETAGS~ property for file-level nodes,
+   and the ~ROAM_TAGS~ property for headline nodes
 6. Replace existing file links with ID links.
 
+** How do I publish my notes with an Internet-friendly graph?
+
+The default graph builder creates a graph with an [[https://orgmode.org/worg/org-contrib/org-protocol.html][org-protocol]]
+handler which is convenient when you're working locally but
+inconvenient when you want to publish your notes for remote access.
+Likewise, it defaults to displaying the graph in Emacs which has the
+exact same caveats.  This problem is solvable in the following way
+using org-mode's native [[https://orgmode.org/manual/Publishing.html][publishing]] capability:
+
+1. configure org-mode to publish your org-roam notes as a project.
+2. create a function that overrides the default org-protocol link
+   creation function(=org-roam-default-link-builder=).
+3. create a hook that's called at the end of graph creation to copy
+   the generated graph to the appropriate place.
+
+The example code below is used to publish to a local directory where a
+separate shell script copies the files to the remote site.
+
+*** Configure org-mode for publishing
+This has two steps:
+1. Setting of a /roam/ project that publishes your notes.
+2. Configuring the /sitemap.html/ generation.
+3. Setting up =org-publish= to generate the graph.
+  
+This will require code like the following:
+#+begin_src emacs-lisp
+  (defun roam-sitemap (title list)
+    (concat "#+OPTIONS: ^:nil author:nil html-postamble:nil\n"
+            "#+SETUPFILE: ./simple_inline.theme\n"
+            "#+TITLE: " title "\n\n"
+            (org-list-to-org list) "\nfile:sitemap.svg"))
+
+  (setq my-publish-time 0)   ; see the next section for context
+  (defun roam-publication-wrapper (plist filename pubdir)
+    (org-roam-graph)
+    (org-html-publish-to-html plist filename pubdir)
+    (setq my-publish-time (cadr (current-time))))
+
+  (setq org-publish-project-alist
+    '(("roam"
+       :base-directory "~/roam"
+       :auto-sitemap t
+       :sitemap-function roam-sitemap
+       :sitemap-title "Roam notes"
+       :publishing-function roam-publication-wrapper
+       :publishing-directory "~/roam-export"
+       :section-number nil
+       :table-of-contents nil
+       :style "<link rel=\"stylesheet\" href=\"../other/mystyle.cs\" type=\"text/css\">")))
+#+end_src
+
+*** Overriding the default link creation function
+The code below will generate a link to the generated html file instead
+of the default org-protocol link.
+#+begin_src emacs-lisp
+  (defun org-roam-custom-link-builder (node)
+    (let ((file (org-roam-node-file node)))
+      (concat (file-name-base file) ".html")))
+
+  (setq org-roam-graph-link-builder 'org-roam-custom-link-builder)
+#+end_src
+
+*** Copying the generated file to the export directory
+The default behavior of =org-roam-graph= is to generate the graph and
+display it in Emacs.  There is an =org-roam-graph-generation-hook=
+available that provides access to the file names so they can be copied
+to the publishing directory.  Example code follows:
+
+#+begin_src emacs-lisp
+  (add-hook 'org-roam-graph-generation-hook
+            (lambda (dot svg) (if (< (- (cadr (current-time)) my-publish-time) 5)
+                                  (progn (copy-file svg "~/roam-export/sitemap.svg" 't)
+                                         (kill-buffer (file-name-nondirectory svg))
+                                         (setq my-publish-time 0)))))
+#+end_src
+** I'm seeing this "Selecting deleted buffer" error. What do I do?
+
+The "selecting deleted buffer" error usually occurs when you don't have a
+working ~emacsql-sqlite~ executable. Org-roam relies on this executable to
+function properly, and doesn't catch this error. This issue is most commonly
+seen on Windows setups. You can browse through the various GitHub issues posted
+about this [[https://github.com/org-roam/org-roam/issues?q=is%3Aissue+selecting+deleted][here]].
+
+To fix this, you can try the following:
+
+1. If on Windows, try replacing your system binary with [[https://github.com/nobiot/emacsql-sqlite.exe][this one]] that has been proven
+   to work
+2. Use the ~emacsql-sqlite3~ option rather than compiling your own emacsql
+   binary (see [[*How to cache][How to cache]]).
+
 * Developer's Guide to Org-roam
 ** Org-roam's Design Principle
 
@@ -1468,7 +1809,7 @@ When GOTO is non-nil, go the note without creating an entry."
 :END:
 
 #+BEGIN_QUOTE
-Copyright (C) 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+Copyright (C) 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 You can redistribute this document and/or modify it under the terms
 of the GNU General Public License as published by the Free Software

extensions/org-roam-dailies.el

@@ -1,14 +1,14 @@
 ;;; org-roam-dailies.el --- Daily-notes for Org-roam -*- coding: utf-8; lexical-binding: t; -*-
 ;;;
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 ;; Copyright © 2020 Leo Vivier <leo.vivier+dev@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;;      Leo Vivier <leo.vivier+dev@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org-roam "2.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org-roam "2.1"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -34,10 +34,9 @@
 ;; each file named after certain date and stored in `org-roam-dailies-directory'.
 ;;
 ;; One can use dailies for various purposes, e.g. journaling, fleeting notes,
-;; scratch notes and whatever else you can came up with.
+;; scratch notes or whatever else you can think of.
 ;;
 ;;; Code:
-(require 'f)
 (require 'dash)
 (require 'org-roam)
 
@@ -62,7 +61,7 @@ This path is relative to `org-roam-directory'."
 (defcustom org-roam-dailies-capture-templates
   `(("d" "default" entry
      "* %?"
-     :if-new (file+head "%<%Y-%m-%d>.org"
+     :target (file+head "%<%Y-%m-%d>.org"
                         "#+title: %<%Y-%m-%d>\n")))
   "Capture templates for daily-notes in Org-roam.
 Note that for daily files to show up in the calendar, they have to be of format
@@ -92,7 +91,7 @@ See `org-roam-capture-templates' for the template documentation."
                                       (function :tag "Template function")))
                         (plist :inline t
                                ;; Give the most common options as checkboxes
-                               :options (((const :format "%v " :if-new)
+                               :options (((const :format "%v " :target)
                                           (choice :tag "Node location"
                                                   (list :tag "File"
                                                         (const :format "" file)
@@ -130,79 +129,103 @@ See `org-roam-capture-templates' for the template documentation."
 ;;; Commands
 ;;;; Today
 ;;;###autoload
-(defun org-roam-dailies-capture-today (&optional goto)
+(defun org-roam-dailies-capture-today (&optional goto keys)
   "Create an entry in the daily-note for today.
-When GOTO is non-nil, go the note without creating an entry."
+When GOTO is non-nil, go the note without creating an entry.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "P")
-  (org-roam-dailies--capture (current-time) goto))
+  (org-roam-dailies--capture (current-time) goto keys))
 
 ;;;###autoload
-(defun org-roam-dailies-goto-today ()
-  "Find the daily-note for today, creating it if necessary."
+(defun org-roam-dailies-goto-today (&optional keys)
+  "Find the daily-note for today, creating it if necessary.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive)
-  (org-roam-dailies-capture-today t))
+  (org-roam-dailies-capture-today t keys))
 
 ;;;; Tomorrow
 ;;;###autoload
-(defun org-roam-dailies-capture-tomorrow (n &optional goto)
+(defun org-roam-dailies-capture-tomorrow (n &optional goto keys)
   "Create an entry in the daily-note for tomorrow.
 
 With numeric argument N, use the daily-note N days in the future.
 
 With a `C-u' prefix or when GOTO is non-nil, go the note without
-creating an entry."
+creating an entry.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "p")
-  (org-roam-dailies--capture (time-add (* n 86400) (current-time)) goto))
+  (org-roam-dailies--capture (time-add (* n 86400) (current-time)) goto keys))
 
 ;;;###autoload
-(defun org-roam-dailies-goto-tomorrow (n)
+(defun org-roam-dailies-goto-tomorrow (n &optional keys)
   "Find the daily-note for tomorrow, creating it if necessary.
 
 With numeric argument N, use the daily-note N days in the
-future."
+future.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "p")
-  (org-roam-dailies-capture-tomorrow n t))
+  (org-roam-dailies-capture-tomorrow n t keys))
 
 ;;;; Yesterday
 ;;;###autoload
-(defun org-roam-dailies-capture-yesterday (n &optional goto)
+(defun org-roam-dailies-capture-yesterday (n &optional goto keys)
   "Create an entry in the daily-note for yesteday.
 
 With numeric argument N, use the daily-note N days in the past.
 
-When GOTO is non-nil, go the note without creating an entry."
+When GOTO is non-nil, go the note without creating an entry.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "p")
-  (org-roam-dailies-capture-tomorrow (- n) goto))
+  (org-roam-dailies-capture-tomorrow (- n) goto keys))
 
 ;;;###autoload
-(defun org-roam-dailies-goto-yesterday (n)
+(defun org-roam-dailies-goto-yesterday (n &optional keys)
   "Find the daily-note for yesterday, creating it if necessary.
 
 With numeric argument N, use the daily-note N days in the
-future."
+future.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "p")
-  (org-roam-dailies-capture-tomorrow (- n) t))
+  (org-roam-dailies-capture-tomorrow (- n) t keys))
 
 ;;;; Date
 ;;;###autoload
-(defun org-roam-dailies-capture-date (&optional goto prefer-future)
+(defun org-roam-dailies-capture-date (&optional goto prefer-future keys)
   "Create an entry in the daily-note for a date using the calendar.
 Prefer past dates, unless PREFER-FUTURE is non-nil.
 With a `C-u' prefix or when GOTO is non-nil, go the note without
-creating an entry."
+creating an entry.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive "P")
   (let ((time (let ((org-read-date-prefer-future prefer-future))
-                (org-read-date t t nil (if goto
-                                           "Find daily-note: "
-                                         "Capture to daily-note: ")))))
-    (org-roam-dailies--capture time goto)))
+                (org-read-date nil t nil (if goto
+                                             "Find daily-note: "
+                                           "Capture to daily-note: ")))))
+    (org-roam-dailies--capture time goto keys)))
 
 ;;;###autoload
-(defun org-roam-dailies-goto-date (&optional prefer-future)
+(defun org-roam-dailies-goto-date (&optional prefer-future keys)
   "Find the daily-note for a date using the calendar, creating it if necessary.
-Prefer past dates, unless PREFER-FUTURE is non-nil."
+Prefer past dates, unless PREFER-FUTURE is non-nil.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
   (interactive)
-  (org-roam-dailies-capture-date t prefer-future))
+  (org-roam-dailies-capture-date t prefer-future keys))
 
 ;;;; Navigation
 (defun org-roam-dailies-goto-next-note (&optional n)
@@ -266,7 +289,7 @@ If FILE is not specified, use the current buffer's file-path."
     (save-match-data
       (and
        (org-roam-file-p path)
-       (f-descendant-of-p path directory)))))
+       (org-roam-descendant-of-p path directory)))))
 
 ;;;###autoload
 (defun org-roam-dailies-find-directory ()
@@ -300,11 +323,16 @@ Return (MONTH DAY YEAR) or nil if not an Org time-string."
 ;;; Capture implementation
 (add-to-list 'org-roam-capture--template-keywords :override-default-time)
 
-(defun org-roam-dailies--capture (time &optional goto)
+(defun org-roam-dailies--capture (time &optional goto keys)
   "Capture an entry in a daily-note for TIME, creating it if necessary.
-When GOTO is non-nil, go the note without creating an entry."
-  (let ((org-roam-directory (expand-file-name org-roam-dailies-directory org-roam-directory)))
+When GOTO is non-nil, go the note without creating an entry.
+
+ELisp programs can set KEYS to a string associated with a template.
+In this case, interactive selection will be bypassed."
+  (let ((org-roam-directory (expand-file-name org-roam-dailies-directory org-roam-directory))
+        (org-roam-dailies-directory "./"))
     (org-roam-capture- :goto (when goto '(4))
+                       :keys keys
                        :node (org-roam-node-create)
                        :templates org-roam-dailies-capture-templates
                        :props (list :override-default-time time)))

extensions/org-roam-export.el

@@ -0,0 +1,75 @@
+;;; org-roam-export.el --- Org-roam org-export tweaks -*- coding: utf-8; lexical-binding: t; -*-
+
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
+
+;; Author: Jethro Kuan <jethrokuan95@gmail.com>
+;; URL: https://github.com/org-roam/org-roam
+;; Keywords: org-mode, roam, convenience
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (org "9.4") (org-roam "2.1"))
+
+;; This file is NOT part of GNU Emacs.
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs; see the file COPYING.  If not, write to the
+;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
+
+;;; Commentary:
+;;
+;; This package provides the necessary changes required to make org-export work out-of-the-box.
+;;
+;; To enable it, run:
+;;
+;;    (require 'org-roam-export)
+;;
+;; The key issue Org's export-to-html functionality has is that it does not respect the ID property, which
+;; Org-roam relies heavily on. This patches the necessary function in ox-html to export ID links correctly,
+;; pointing to the correct place.
+;;
+;;; Code:
+(require 'ox-html)
+
+(defun org-roam-export--org-html--reference (datum info &optional named-only)
+  "Org-roam's patch for `org-html--reference' to support ID link export.
+See `org-html--reference' for DATUM, INFO and NAMED-ONLY."
+  (let* ((type (org-element-type datum))
+         (user-label
+          (org-element-property
+           (pcase type
+             ((or `headline `inlinetask) :CUSTOM_ID)
+             ((or `radio-target `target) :value)
+             (_ :name))
+           datum))
+         (user-label
+          (or user-label
+              (when-let ((path (org-element-property :ID datum)))
+                ;; see `org-html-link' for why we use "ID-"
+                ;; (search for "ID-" in ox-html.el)
+                (concat "ID-" path)))))
+    (cond
+     ((and user-label
+           (or (plist-get info :html-prefer-user-labels)
+               (memq type '(headline inlinetask))))
+      user-label)
+     ((and named-only
+           (not (memq type '(headline inlinetask radio-target target)))
+           (not user-label))
+      nil)
+     (t
+      (org-export-get-reference datum info)))))
+
+(advice-add 'org-html--reference :override #'org-roam-export--org-html--reference)
+
+(provide 'org-roam-export)
+;;; org-roam-export.el ends here

extensions/org-roam-graph.el

@@ -1,11 +1,11 @@
 ;;; org-roam-graph.el --- Basic graphing functionality for Org-roam -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
+;; Version: 2.2.1
 ;; Package-Requires: ((emacs "26.1") (org "9.4") (org-roam "2.1"))
 
 ;; This file is NOT part of GNU Emacs.
@@ -81,7 +81,7 @@ Example:
                ("fillcolor"  . "#EEEEEE")
                ("color"      . "#C9C9C9")
                ("fontcolor"  . "#0A97A6")))
-    ("https" . (("shape"      . "rounded,filled")
+    ("https" . (("style"      . "rounded,filled")
                 ("fillcolor"  . "#EEEEEE")
                 ("color"      . "#C9C9C9")
                 ("fontcolor"  . "#0A97A6"))))
@@ -113,6 +113,25 @@ All other values including nil will have no effect."
           (const :tag "no" nil))
   :group 'org-roam)
 
+(defcustom org-roam-graph-link-builder 'org-roam-org-protocol-link-builder
+  "Function used to build the Org-roam graph links.
+Given a node name, return a string to be used for the link fed to
+the graph generation utility."
+  :type 'function
+  :group 'org-roam)
+
+(defcustom org-roam-graph-generation-hook nil
+  "Functions to run after the graph has been generated.
+Each function is called with two arguments: the filename
+containing the graph generation tool, and the generated graph."
+  :type 'hook
+  :group 'org-roam)
+
+(defun org-roam-org-protocol-link-builder (node)
+  "Default org-roam link builder.  Generate an org-protocol link using NODE."
+  (concat "org-protocol://roam-node?node="
+          (url-hexify-string (org-roam-node-id node))))
+
 ;;; Interactive command
 ;;;###autoload
 (defun org-roam-graph (&optional arg node)
@@ -147,13 +166,14 @@ CALLBACK is passed the graph file as its sole argument."
          (temp-graph (make-temp-file "graph." nil (concat "." org-roam-graph-filetype))))
     (org-roam-message "building graph")
     (make-process
-     :name "*org-roam-graph--build-process*"
-     :buffer "*org-roam-graph--build-process*"
+     :name "*org-roam-graph*"
+     :buffer " *org-roam-graph*"
      :command `(,org-roam-graph-executable ,temp-dot "-T" ,org-roam-graph-filetype "-o" ,temp-graph)
      :sentinel (when callback
                  (lambda (process _event)
                    (when (= 0 (process-exit-status process))
-                     (funcall callback temp-graph)))))))
+                     (progn (funcall callback temp-graph)
+                            (run-hook-with-args 'org-roam-graph-generation-hook temp-dot temp-graph))))))))
 
 (defun org-roam-graph--dot (&optional edges all-nodes)
   "Build the graphviz given the EDGES of the graph.
@@ -246,12 +266,11 @@ Handles both Org-roam nodes, and string nodes (e.g. urls)."
                 (org-roam-quote-string
                  (pcase org-roam-graph-shorten-titles
                    (`truncate (truncate-string-to-width title org-roam-graph-max-title-length nil nil "..."))
-                   (`wrap (s-word-wrap org-roam-graph-max-title-length title))
+                   (`wrap (org-roam-word-wrap org-roam-graph-max-title-length title))
                    (_ title)))))
           (setq node-id (org-roam-node-id node)
                 node-properties `(("label"   . ,shortened-title)
-                                  ("URL"     . ,(concat "org-protocol://roam-node?node="
-                                                        (url-hexify-string (org-roam-node-id node))))
+                                  ("URL"     . ,(funcall org-roam-graph-link-builder node))
                                   ("tooltip" . ,(xml-escape-string title)))))
       (setq node-id node
             node-properties (append `(("label" . ,(concat type ":" node)))

extensions/org-roam-overlay.el

@@ -1,11 +1,11 @@
 ;;; org-roam-overlay.el --- Link overlay for [id:] links to Org-roam nodes -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
+;; Version: 2.2.1
 ;; Package-Requires: ((emacs "26.1") (org "9.4") (org-roam "2.1"))
 
 ;; This file is NOT part of GNU Emacs.
@@ -27,8 +27,8 @@
 
 ;;; Commentary:
 ;;
-;; This extension provides allows to render [[id:]] links that don't have an
-;; asscoiated descriptor with an overlay that displays the node's current title.
+;; This extension allows to render [[id:]] links that don't have an associated
+;; descriptor with an overlay that displays the node's current title.
 ;;
 ;;; Code:
 (require 'org-roam)

extensions/org-roam-protocol.el

@@ -1,10 +1,10 @@
 ;;; org-roam-protocol.el --- Protocol handler for roam:// links  -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
+;; Version: 2.2.1
 ;; Package-Requires: ((emacs "26.1") (org "9.4") (org-roam "2.1"))
 
 ;; This file is NOT part of GNU Emacs.
@@ -48,7 +48,7 @@
 
 (defcustom org-roam-capture-ref-templates
   '(("r" "ref" plain "%?"
-     :if-new (file+head "${slug}.org"
+     :target (file+head "${slug}.org"
                         "#+title: ${title}")
      :unnarrowed t))
   "The Org-roam templates used during a capture from the roam-ref protocol.
@@ -77,7 +77,7 @@ See `org-roam-capture-templates' for the template documentation."
                                       (function :tag "Template function")))
                         (plist :inline t
                                ;; Give the most common options as checkboxes
-                               :options (((const :format "%v " :if-new)
+                               :options (((const :format "%v " :target)
                                           (choice :tag "Node location"
                                                   (list :tag "File"
                                                         (const :format "" file)
@@ -141,12 +141,13 @@ It opens or creates a note with the given ref.
                                                               (plist-get info :ref)))
                         :initial (or (plist-get info :body) ""))
   (raise-frame)
-  (org-roam-capture-
-   :keys (plist-get info :template)
-   :node (org-roam-node-create :title (plist-get info :title))
-   :info (list :ref (plist-get info :ref)
-               :body (plist-get info :body))
-   :templates org-roam-capture-ref-templates)
+  (let ((org-capture-link-is-already-stored t))
+    (org-roam-capture-
+     :keys (plist-get info :template)
+     :node (org-roam-node-create :title (plist-get info :title))
+     :info (list :ref (plist-get info :ref)
+                 :body (plist-get info :body))
+     :templates org-roam-capture-ref-templates))
   nil)
 
 (defun org-roam-protocol-open-node (info)
@@ -168,25 +169,6 @@ org-protocol://roam-node?node=uuid"
 (push '("org-roam-node"  :protocol "roam-node"   :function org-roam-protocol-open-node)
       org-protocol-protocol-alist)
 
-;;; Capture implementation
-(add-hook 'org-roam-capture-preface-hook #'org-roam-protocol--try-capture-to-ref-h)
-(defun org-roam-protocol--try-capture-to-ref-h ()
-  "Try to capture to an existing node that match the ref."
-  (when-let ((node (and (plist-get org-roam-capture--info :ref)
-                        (org-roam-node-from-ref
-                         (plist-get org-roam-capture--info :ref)))))
-    (set-buffer (org-capture-target-buffer (org-roam-node-file node)))
-    (goto-char (org-roam-node-point node))
-    (widen)
-    (org-roam-node-id node)))
-
-(add-hook 'org-roam-capture-new-node-hook #'org-roam-protocol--insert-captured-ref-h)
-(defun org-roam-protocol--insert-captured-ref-h ()
-  "Insert the ref if any."
-  (when-let ((ref (plist-get org-roam-capture--info :ref)))
-    (org-roam-ref-add ref)))
-
-
 (provide 'org-roam-protocol)
 
 ;;; org-roam-protocol.el ends here

github-eldev

@@ -0,0 +1,11 @@
+#! /bin/sh
+set -e
+
+ELDEV_BIN_DIR=~/.local/bin
+
+# `$GITHUB_PATH' is a magic file which contents is translated to environment variable `$PATH'.
+echo "$ELDEV_BIN_DIR" >> $GITHUB_PATH
+
+mkdir -p $ELDEV_BIN_DIR
+curl -fsSL https://raw.githubusercontent.com/doublep/eldev/f111d19cda305e5e8fcb70a5675b87173041cb68/bin/eldev > $ELDEV_BIN_DIR/eldev
+chmod a+x $ELDEV_BIN_DIR/eldev

org-roam-capture.el

@@ -1,12 +1,12 @@
 ;;; org-roam-capture.el --- Capture functionality -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -40,7 +40,7 @@
 ;;; Options
 (defcustom org-roam-capture-templates
   '(("d" "default" plain "%?"
-     :if-new (file+head "%<%Y%m%d%H%M%S>-${slug}.org"
+     :target (file+head "%<%Y%m%d%H%M%S>-${slug}.org"
                         "#+title: ${title}\n")
      :unnarrowed t))
   "Templates for the creation of new entries within Org-roam.
@@ -92,30 +92,41 @@ template     The template for creating the capture item.
              in order to get a template from a file, or dynamically
              from a function.
 
-The template contains a compulsory :if-new property. This determines the
-location of the new node. The :if-new property contains a list, supporting
-the following options:
+The template contains a compulsory :target property. The :target property
+contains a list, where:
+  - The first element indicates the type of the target.
+  - The second element indicates the location of the captured node.
+  - And the rest of the list indicate the prefilled template, that will be
+    inserted and the position of the point will be adjusted for.
+    This behavior varies from type to type.
+
+The following options are supported for the :target property:
 
    (file \"path/to/file\")
        The file will be created, and prescribed an ID.
 
    (file+head \"path/to/file\" \"head content\")
        The file will be created, prescribed an ID, and head content will be
-       inserted into the file.
+       inserted if the node is a newly captured one.
 
    (file+olp \"path/to/file\" (\"h1\" \"h2\"))
-       The file will be created, prescribed an ID. The OLP (h1, h2) will be
-       created, and the point placed after.
+       The file will be created, prescribed an ID. If the file doesn't contain
+       the outline path (h1, h2), it will be automatically created. The point
+       will be adjusted to the last element in the OLP.
 
    (file+head+olp \"path/to/file\" \"head content\" (\"h1\" \"h2\"))
        The file will be created, prescribed an ID. Head content will be
-       inserted at the start of the file. The OLP (h1, h2) will be created,
-       and the point placed after.
+       inserted at the start of the file if the node is a newly captured one.
+       If the file doesn't contain the outline path (h1, h2), it will be
+       automatically created. The point will be adjusted to the last element in
+       the OLP.
 
-   (file+datetree \"path/to/file\" day)
-       The file will be created, prescribed an ID. Head content will be
-       inserted at the start of the file. The datetree will be created,
-       available options are day, week, month.
+   (file+datetree \"path/to/file\" tree-type)
+       The file will be created, prescribed an ID. A date based outline path
+       will be created for today's date. The tree-type can be one of the
+       following symbols: day, week or month. The point will adjusted to the
+       last element in the tree. To prompt for date instead of using today's,
+       use the :time-prompt property.
 
    (node \"title or alias or ID of an existing node\")
        The point will be placed for an existing node, based on either, its
@@ -194,7 +205,8 @@ be replaced with content and expanded:
               introduced with %[pathname] are expanded this way.  Since this
               happens after expanding non-interactive %-escapes, those can
               be used to fill the expression.
-  %<...>      The result of `format-time-string' on the ... format specification.
+  %<...>      The result of `format-time-string' on the ... format
+              specification.
   %t          Time stamp, date only.  The time stamp is the current time,
               except when called from agendas with `\\[org-agenda-capture]' or
               with `org-capture-use-agenda-date' set.
@@ -289,7 +301,7 @@ streamlined user experience in Org-roam."
                                       (function :tag "Template function")))
                         (plist :inline t
                                ;; Give the most common options as checkboxes
-                               :options (((const :format "%v " :if-new)
+                               :options (((const :format "%v " :target)
                                           (choice :tag "Node location"
                                                   (list :tag "File"
                                                         (const :format "" file)
@@ -377,7 +389,7 @@ during the Org-roam capture process.")
 This variable is populated dynamically, and is only non-nil
 during the Org-roam capture process.")
 
-(defconst org-roam-capture--template-keywords (list :if-new :id :link-description :call-location
+(defconst org-roam-capture--template-keywords (list :target :id :link-description :call-location
                                                     :region)
   "Keywords used in `org-roam-capture-templates' specific to Org-roam.")
 
@@ -395,6 +407,8 @@ TEMPLATES is a list of org-roam templates."
           (mapcar (lambda (template)
                     (org-roam-capture--convert-template template props))
                   (or templates org-roam-capture-templates)))
+         (_ (setf (org-roam-node-id node) (or (org-roam-node-id node)
+                                              (org-id-new))))
          (org-roam-capture--node node)
          (org-roam-capture--info info))
     (when (and (not keys)
@@ -433,36 +447,33 @@ the capture)."
   "Get the value for KEYWORD from the `org-roam-capture-template'."
   (plist-get (plist-get org-capture-plist :org-roam) keyword))
 
-(defun org-roam-capture--put (&rest stuff)
-  "Put properties from STUFF into the `org-roam-capture-template'."
+(defun org-roam-capture--put (prop value)
+  "Set property PROP to VALUE in the `org-roam-capture-template'."
   (let ((p (plist-get org-capture-plist :org-roam)))
-    (while stuff
-      (setq p (plist-put p (pop stuff) (pop stuff))))
     (setq org-capture-plist
-          (plist-put org-capture-plist :org-roam p))))
+          (plist-put org-capture-plist
+                     :org-roam
+                     (plist-put p prop value)))))
 
 ;;;; Capture target
 (defun org-roam-capture--prepare-buffer ()
   "Prepare the capture buffer for the current Org-roam based capture template.
 This function will initialize and setup the capture buffer,
-create the target node (`:if-new') if it doesn't exist, and place
-the point for further processing by `org-capture'.
+position the point to the current :target (and if necessary,
+create it if it doesn't exist), and place the point for further
+processing by `org-capture'.
 
 Note: During the capture process this function is run by
 `org-capture-set-target-location', as a (function ...) based
 capture target."
   (let ((id (cond ((run-hook-with-args-until-success 'org-roam-capture-preface-hook))
-                  ((and (org-roam-node-file org-roam-capture--node)
-                        (org-roam-node-point org-roam-capture--node))
-                   (set-buffer (org-capture-target-buffer (org-roam-node-file org-roam-capture--node)))
-                   (goto-char (org-roam-node-point org-roam-capture--node))
-                   (widen)
-                   (org-roam-node-id org-roam-capture--node))
-                  (t
-                   (org-roam-capture--setup-target-location)))))
+                  (t (org-roam-capture--setup-target-location)))))
     (org-roam-capture--adjust-point-for-capture-type)
-    (org-capture-put :template
-                     (org-roam-capture--fill-template (org-capture-get :template)))
+    (let ((template (org-capture-get :template)))
+      (when (stringp template)
+        (org-capture-put
+         :template
+         (org-roam-capture--fill-template template))))
     (org-roam-capture--put :id id)
     (org-roam-capture--put :finalize (or (org-capture-get :finalize)
                                          (org-roam-capture--get :finalize)))))
@@ -471,22 +482,17 @@ capture target."
   "Initialize the buffer, and goto the location of the new capture.
 Return the ID of the location."
   (let (p new-file-p)
-    (pcase (or (org-roam-capture--get :if-new)
-               (user-error "Template needs to specify `:if-new'"))
+    (pcase (org-roam-capture--get-target)
       (`(file ,path)
-       (setq path (expand-file-name
-                   (string-trim (org-roam-capture--fill-template path t))
-                   org-roam-directory))
-       (setq new-file-p (org-roam-capture--new-file-p path))
+       (setq path (org-roam-capture--target-truepath path)
+             new-file-p (org-roam-capture--new-file-p path))
        (when new-file-p (org-roam-capture--put :new-file path))
        (set-buffer (org-capture-target-buffer path))
        (widen)
        (setq p (goto-char (point-min))))
       (`(file+olp ,path ,olp)
-       (setq path (expand-file-name
-                   (string-trim (org-roam-capture--fill-template path t))
-                   org-roam-directory))
-       (setq new-file-p (org-roam-capture--new-file-p path))
+       (setq path (org-roam-capture--target-truepath path)
+             new-file-p (org-roam-capture--new-file-p path))
        (when new-file-p (org-roam-capture--put :new-file path))
        (set-buffer (org-capture-target-buffer path))
        (setq p (point-min))
@@ -494,33 +500,27 @@ Return the ID of the location."
          (goto-char m))
        (widen))
       (`(file+head ,path ,head)
-       (setq path (expand-file-name
-                   (string-trim (org-roam-capture--fill-template path t))
-                   org-roam-directory))
-       (setq new-file-p (org-roam-capture--new-file-p path))
+       (setq path (org-roam-capture--target-truepath path)
+             new-file-p (org-roam-capture--new-file-p path))
        (set-buffer (org-capture-target-buffer path))
        (when new-file-p
          (org-roam-capture--put :new-file path)
-         (insert (org-roam-capture--fill-template head t)))
+         (insert (org-roam-capture--fill-template head 'ensure-newline)))
        (widen)
        (setq p (goto-char (point-min))))
       (`(file+head+olp ,path ,head ,olp)
-       (setq path (expand-file-name
-                   (string-trim (org-roam-capture--fill-template path t))
-                   org-roam-directory))
-       (setq new-file-p (org-roam-capture--new-file-p path))
+       (setq path (org-roam-capture--target-truepath path)
+             new-file-p (org-roam-capture--new-file-p path))
        (set-buffer (org-capture-target-buffer path))
        (widen)
        (when new-file-p
          (org-roam-capture--put :new-file path)
-         (insert (org-roam-capture--fill-template head t)))
+         (insert (org-roam-capture--fill-template head 'ensure-newline)))
        (setq p (point-min))
        (let ((m (org-roam-capture-find-or-create-olp olp)))
          (goto-char m)))
       (`(file+datetree ,path ,tree-type)
-       (setq path (expand-file-name
-                   (string-trim (org-roam-capture--fill-template path t))
-                   org-roam-directory))
+       (setq path (org-roam-capture--target-truepath path))
        (require 'org-datetree)
        (widen)
        (set-buffer (org-capture-target-buffer path))
@@ -574,13 +574,29 @@ Return the ID of the location."
     ;; caller.
     (save-excursion
       (goto-char p)
-      (when-let* ((node org-roam-capture--node)
-                  (id (org-roam-node-id node)))
-        (org-entry-put p "ID" id))
+      (if-let ((id (org-entry-get p "ID")))
+          (setf (org-roam-node-id org-roam-capture--node) id)
+        (org-entry-put p "ID" (org-roam-node-id org-roam-capture--node)))
       (prog1
-          (org-id-get-create)
+          (org-id-get)
         (run-hooks 'org-roam-capture-new-node-hook)))))
 
+(defun org-roam-capture--get-target ()
+  "Get the current capture :target for the capture template in use."
+  (or (org-roam-capture--get :target)
+      (user-error "Template needs to specify `:target'")))
+
+(defun org-roam-capture--target-truepath (path)
+  "From PATH get the correct path to the current capture target and return it.
+PATH is a string that can optionally contain templated text in
+it."
+  (or (org-roam-node-file org-roam-capture--node)
+      (thread-first
+        path
+        (org-roam-capture--fill-template)
+        (string-trim)
+        (expand-file-name org-roam-directory))))
+
 (defun org-roam-capture--new-file-p (path)
   "Return t if PATH is for a new file with no visiting buffer."
   (not (or (file-exists-p path)
@@ -602,6 +618,7 @@ you can catch it with `condition-case'."
     (org-with-wide-buffer
      (goto-char start)
      (dolist (heading olp)
+       (setq heading (org-roam-capture--fill-template heading))
        (let ((re (format org-complex-heading-regexp-format
                          (regexp-quote heading)))
              (cnt 0))
@@ -663,6 +680,24 @@ the current value of `point'."
             (goto-char (org-entry-end-position))))))))
   (point))
 
+;;; Capture implementation
+(add-hook 'org-roam-capture-preface-hook #'org-roam-capture--try-capture-to-ref-h)
+(defun org-roam-capture--try-capture-to-ref-h ()
+  "Try to capture to an existing node that match the ref."
+  (when-let ((node (and (plist-get org-roam-capture--info :ref)
+                        (org-roam-node-from-ref
+                         (plist-get org-roam-capture--info :ref)))))
+    (set-buffer (org-capture-target-buffer (org-roam-node-file node)))
+    (goto-char (org-roam-node-point node))
+    (widen)
+    (org-roam-node-id node)))
+
+(add-hook 'org-roam-capture-new-node-hook #'org-roam-capture--insert-captured-ref-h)
+(defun org-roam-capture--insert-captured-ref-h ()
+  "Insert the ref if any."
+  (when-let ((ref (plist-get org-roam-capture--info :ref)))
+    (org-roam-ref-add ref)))
+
 ;;;; Finalizers
 (add-hook 'org-capture-prepare-finalize-hook #'org-roam-capture--install-finalize-h)
 (defun org-roam-capture--install-finalize-h ()
@@ -675,11 +710,14 @@ the current value of `point'."
   (when-let ((region (org-roam-capture--get :region)))
     (org-roam-unshield-region (car region) (cdr region)))
   (if org-note-abort
-      (when-let ((new-file (org-roam-capture--get :new-file)))
-        (org-roam-message "Deleting file for aborted capture %s" new-file)
+      (when-let ((new-file (org-roam-capture--get :new-file))
+                 (_ (yes-or-no-p "Delete file for aborted capture?")))
         (when (find-buffer-visiting new-file)
           (kill-buffer (find-buffer-visiting new-file)))
         (delete-file new-file))
+    (when-let* ((buffer (plist-get org-capture-plist :buffer))
+                (file (buffer-file-name buffer)))
+      (org-id-add-location (org-roam-capture--get :id) file))
     (when-let* ((finalize (org-roam-capture--get :finalize))
                 (org-roam-finalize-fn (intern (concat "org-roam-capture--finalize-"
                                                       (symbol-name finalize)))))
@@ -706,32 +744,56 @@ This function is to be called in the Org-capture finalization process."
         (delete-region (car region) (cdr region))
         (set-marker (car region) nil)
         (set-marker (cdr region) nil))
-      (org-with-point-at mkr
-        (insert (org-link-make-string (concat "id:" (org-roam-capture--get :id))
-                                      (org-roam-capture--get :link-description)))))))
+      (let* ((id (org-roam-capture--get :id))
+             (description (org-roam-capture--get :link-description))
+             (link (org-link-make-string (concat "id:" id)
+                                         description)))
+        (if (eq (point) (marker-position mkr))
+            (insert link)
+          (org-with-point-at mkr
+            (insert link)))
+        (run-hook-with-args 'org-roam-post-node-insert-hook
+                            id
+                            description)))))
 
 ;;;; Processing of the capture templates
-(defun org-roam-capture--fill-template (template &optional org-capture-p)
+(defun org-roam-capture--fill-template (template &optional ensure-newline)
   "Expand TEMPLATE and return it.
-It expands ${var} occurrences in TEMPLATE. When ORG-CAPTURE-P,
-also run Org-capture's template expansion."
-  (funcall (if org-capture-p #'org-capture-fill-template #'identity)
-           (org-roam-format-template
-            template
-            (lambda (key default-val)
-              (let ((fn (intern key))
-                    (node-fn (intern (concat "org-roam-node-" key)))
-                    (ksym (intern (concat ":" key))))
-                (cond
-                 ((fboundp fn)
-                  (funcall fn org-roam-capture--node))
-                 ((fboundp node-fn)
-                  (funcall node-fn org-roam-capture--node))
-                 ((plist-get org-roam-capture--info ksym)
-                  (plist-get org-roam-capture--info ksym))
-                 (t (let ((r (completing-read (format "%s: " key) nil nil nil default-val)))
-                      (plist-put org-roam-capture--info ksym r)
-                      r))))))))
+It expands ${var} occurrences in TEMPLATE, and then runs
+org-capture's template expansion.
+When ENSURE-NEWLINE, always ensure there's a newline behind."
+  (let* ((template (if (functionp template)
+                       (funcall template)
+                     template))
+         (template-whitespace-content (org-roam-whitespace-content template)))
+    (setq template
+          (org-roam-format-template
+           template
+           (lambda (key default-val)
+             (let ((fn (intern key))
+                   (node-fn (intern (concat "org-roam-node-" key)))
+                   (ksym (intern (concat ":" key))))
+               (cond
+                ((fboundp fn)
+                 (funcall fn org-roam-capture--node))
+                ((fboundp node-fn)
+                 (funcall node-fn org-roam-capture--node))
+                ((plist-get org-roam-capture--info ksym)
+                 (plist-get org-roam-capture--info ksym))
+                (t (let ((r (read-from-minibuffer (format "%s: " key) default-val)))
+                     (plist-put org-roam-capture--info ksym r)
+                     r)))))))
+    ;; WARNING:
+    ;; `org-capture-fill-template' fills the template, but post-processes whitespace such that the resultant
+    ;; template does not start with any whitespace, and only ends with a single newline
+    ;;
+    ;; Instead, we restore the whitespace in the original template.
+    (setq template (replace-regexp-in-string "\n$" "" (org-capture-fill-template template)))
+    (when (and ensure-newline
+               (string-equal template-whitespace-content ""))
+      (setq template-whitespace-content "\n"))
+    (setq template (concat template template-whitespace-content))
+    template))
 
 (defun org-roam-capture--convert-template (template &optional props)
   "Convert TEMPLATE from Org-roam syntax to `org-capture-templates' syntax.

org-roam-compat.el

@@ -1,11 +1,11 @@
 ;;; org-roam-compat.el --- Backward compatibility code -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
+;; Version: 2.2.1
 ;; Package-Requires: ((emacs "26.1"))
 
 ;; This file is NOT part of GNU Emacs.
@@ -31,6 +31,8 @@
 ;; Emacsen and Org-roam versions.
 ;;
 ;;; Code:
+(require 'org-roam)
+
 ;;; Backports
 ;; REVIEW Remove when 26.x support is dropped. This is exact the same as
 ;; `directory-files-recursively' from Emacs 26, but with FOLLOW-SYMLINKS
@@ -99,35 +101,33 @@ recursion."
 (advice-add #'org-id-add-location :around #'org-roam--handle-absent-org-id-locations-file-a)
 (defun org-roam--handle-absent-org-id-locations-file-a (fn &rest args)
   "Gracefully handle errors related to absence of `org-id-locations-file'.
-FN is `org-id-locations-file' that comes from advice and ARGS are
+FN is `org-id-add-location' that comes from advice and ARGS are
 passed to it."
-  (let (result)
-    ;; Use `unwind-protect' over `condition-case' because `org-id' can produce various other errors, but all
-    ;; of its errors are generic ones, so trapping all of them isn't a good idea and preserving the correct
-    ;; backtrace is valuable.
-    (unwind-protect (setq result (apply fn args))
-      (unless result
-        (unless org-id-locations
-          ;; Pre-allocate the hash table to avoid weird access related errors during the regeneration.
-          (setq org-id-locations (make-hash-table :type 'equal)))
-        ;; `org-id' makes the assumption that `org-id-locations-file' will be stored in `user-emacs-directory'
-        ;; which always exist if you have Emacs, so it uses `with-temp-file' to write to the file. However,
-        ;; the users *do* change the path to this file and `with-temp-file' unable to create the file, if the
-        ;; path to it consists of directories that don't exist. We'll have to handle this ourselves.
-        (unless (file-exists-p (file-truename org-id-locations-file))
-          ;; If permissions allow that, try to create the user specified directory path to
-          ;; `org-id-locations-file' ourselves.
-          (condition-case _err
-              (progn (org-roam-message (concat "`org-id-locations-file' (%s) doesn't exist. "
-                                               "Trying to regenerate it (this may take a while)...")
-                                       org-id-locations-file)
-                     (make-directory (file-name-directory (file-truename org-id-locations-file)))
-                     (org-roam-update-org-id-locations)
-                     (apply fn args))
-            ;; In case of failure (lack of permissions), we'll patch it to at least handle the current session
-            ;; without errors.
-            (file-error (org-roam-message "Failed to regenerate `org-id-locations-file'")
-                        (lwarn 'org-roam :error "
+  (condition-case err
+      (apply fn args)
+    ;; `org-id' makes the assumption that `org-id-locations-file' will be stored in `user-emacs-directory'
+    ;; which always exist if you have Emacs, so it uses `with-temp-file' to write to the file. However, the
+    ;; users *do* change the path to this file and `with-temp-file' unable to create the file, if the path to
+    ;; it consists of directories that don't exist. We'll have to handle this ourselves.
+    (error
+     (advice-remove 'org-id-add-location #'org-roam--handle-absent-org-id-locations-file-a)
+     (if (file-exists-p (file-truename org-id-locations-file))
+         (signal (car err) (cdr err))
+       ;; Pre-allocate the hash table to avoid weird access related errors during the regeneration.
+       (or org-id-locations (setq org-id-locations (make-hash-table :test 'equal)))
+       ;; If permissions allow that, try to create the user specified directory path to
+       ;; `org-id-locations-file' ourselves.
+       (condition-case _err
+           (progn (org-roam-message (concat "`org-id-locations-file' (%s) doesn't exist. "
+                                            "Trying to regenerate it (this may take a while)...")
+                                    org-id-locations-file)
+                  (make-directory (file-name-directory (file-truename org-id-locations-file)))
+                  (org-roam-update-org-id-locations)
+                  (apply fn args))
+         ;; In case of failure (lack of permissions), we'll patch it to at least handle the current session
+         ;; without errors.
+         (file-error (org-roam-message "Failed to regenerate `org-id-locations-file'")
+                     (lwarn 'org-roam :error "
 --------
 WARNING: `org-id-locations-file' (%s) doesn't exist!
          Org-roam is unable to create it for you.
@@ -151,10 +151,32 @@ allows to keep linking with \"id:\" links within the current
 `org-roam-directory' to headings and files that are excluded from
 identification (e.g. with \"ROAM_EXCLUDE\" property) as Org-roam
 nodes." org-id-locations-file)
-                        (setq org-id-locations-file
-                              (expand-file-name ".orgids" (file-truename org-roam-directory)))
-                        (apply fn args)))))
-      result)))
+                     (setq org-id-locations-file
+                           (expand-file-name ".orgids" (file-truename org-roam-directory)))
+                     (apply fn args)))))))
+
+;;;; Deprecated :if-new capture template keyword
+(with-eval-after-load 'org-roam-capture
+  (add-to-list 'org-roam-capture--template-keywords :if-new)
+
+  (let ((inhibit-warning-p t)) ; REVIEW Set this to nil close to next major release
+    (advice-add 'org-roam-capture--get-target :around #'org-roam-capture--get-if-new-target-a)
+    (defun org-roam-capture--get-if-new-target-a (fn &rest args)
+      "Get the current capture target using deprecated :if-new property."
+      (if-let ((target (org-roam-capture--get :if-new)))
+          (prog1 target
+            (unless inhibit-warning-p
+              (lwarn 'org-roam-capture :warning
+                     (mapconcat
+                      #'identity
+                      ["`:if-new' property is deprecated in favor of `:target'."
+                       "This warning will popup once per each session. In order to get"
+                       "rid of it, rename all the references to the `:if-new' property"
+                       "in your capture templates to `:target'."]
+                      "\n"))
+              ;; Don't irritate the user too much. Displaying the warning once per session should be enough.
+              (setq inhibit-warning-p t)))
+        (apply fn args)))))
 
 ;;; Obsolete aliases (remove after next major release)
 (define-obsolete-function-alias
@@ -199,6 +221,18 @@ nodes." org-id-locations-file)
   'org-roam-dailies-find-date
   'org-roam-dailies-goto-date "org-roam 2.0")
 
+(define-obsolete-function-alias
+  'org-roam-add-property
+  'org-roam-property-add "org-roam 2.1")
+
+(define-obsolete-function-alias
+  'org-roam-remove-property
+  'org-roam-property-remove "org-roam 2.1")
+
+(define-obsolete-variable-alias
+  'org-roam-mode-section-functions
+  'org-roam-mode-sections "org-roam 2.2.0")
+
 ;;; Obsolete functions
 (make-obsolete 'org-roam-get-keyword 'org-collect-keywords "org-roam 2.0")
 

org-roam-db.el

@@ -1,12 +1,12 @@
 ;;; org-roam-db.el --- Org-roam database API -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -34,7 +34,39 @@
 (defvar org-outline-path-cache)
 
 ;;; Options
-(defcustom org-roam-db-location (expand-file-name "org-roam.db" user-emacs-directory)
+(defcustom org-roam-database-connector 'sqlite
+  "The database connector used by Org-roam.
+This must be set before `org-roam' is loaded.  To use an alternative
+connector you must install the respective package explicitly.
+The default is `sqlite', which uses the `emacsql-sqlite' library
+that is being maintained in the same repository as `emacsql'
+itself.
+If you are using Emacs 29, then the recommended connector is
+`sqlite-builtin', which uses the new builtin support for SQLite.
+You need to install the `emacsql-sqlite-builtin' package to use
+this connector.
+If you are using an older Emacs release, then the recommended
+connector is `sqlite-module', which uses the module provided by
+the `sqlite3' package.  This is very similar to the previous
+connector and the built-in support in Emacs 29 derives from this
+module.  You need to install the `emacsql-sqlite-module' package
+to use this connector.
+For the time being `libsqlite3' is still supported.  Do not use
+this, it is an older version of the `sqlite-module' connector
+from before the connector and the package were renamed.
+For the time being `sqlite3' is also supported.  Do not use this.
+This uses the third-party `emacsql-sqlite3' package, which uses
+the official `sqlite3' cli tool, which is not intended
+to be used like this.  See https://nullprogram.com/blog/2014/02/06/."
+  :package-version '(forge . "0.3.0")
+  :group 'forge
+  :type '(choice (const sqlite)
+                 (const sqlite-builtin)
+                 (const sqlite-module)
+                 (const :tag "libsqlite3 (OBSOLETE)" libsqlite3)
+                 (const :tag "sqlite3 (BROKEN)" sqlite3)))
+
+(defcustom org-roam-db-location (locate-user-emacs-file "org-roam.db")
   "The path to file where the Org-roam database is stored.
 
 It is the user's responsibility to set this correctly, especially
@@ -78,14 +110,34 @@ slow."
   :type 'boolean
   :group 'org-roam)
 
-;;; Variables
-(defconst org-roam-db-version 16)
+(defcustom org-roam-db-extra-links-elements '(node-property keyword)
+  "The list of Org element types to include for parsing by Org-roam.
 
-;; TODO Rename this
-(defconst org-roam--sqlite-available-p
-  (with-demoted-errors "Org-roam initialization: %S"
-    (emacsql-sqlite-ensure-binary)
-    t))
+By default, when parsing Org's AST, links within keywords and
+property drawers are not parsed as links. Sometimes however, it
+is desirable to parse and cache these links (e.g. hiding links in
+a property drawer)."
+  :package-version '(org-roam . "2.2.0")
+  :group 'org-roam
+  :type '(set (const :tag "keywords" keyword)
+              (const :tag "property drawers" node-property)))
+
+(defcustom org-roam-db-extra-links-exclude-keys '((node-property . ("ROAM_REFS"))
+                                                  (keyword . ("transclude")))
+  "Keys to ignore when mapping over links.
+
+The car of the association list is the Org element type (e.g.
+keyword). The cdr is a list of case-insensitive strings to
+exclude from being treated as links.
+
+For example, we use this to prevent self-referential links in
+ROAM_REFS."
+  :package-version '(org-roam . "2.2.0")
+  :group 'org-roam
+  :type '(alist))
+
+;;; Variables
+(defconst org-roam-db-version 18)
 
 (defvar org-roam-db--connection (make-hash-table :test #'equal)
   "Database connection to Org-roam database.")
@@ -93,9 +145,39 @@ slow."
 ;;; Core Functions
 (defun org-roam-db--get-connection ()
   "Return the database connection, if any."
-  (gethash (expand-file-name org-roam-directory)
+  (gethash (expand-file-name (file-name-as-directory org-roam-directory))
            org-roam-db--connection))
 
+(declare-function emacsql-sqlite "ext:emacsql-sqlite")
+(declare-function emacsql-sqlite3 "ext:emacsql-sqlite3")
+(declare-function emacsql-libsqlite3 "ext:emacsql-libsqlite3")
+(declare-function emacsql-sqlite-builtin "ext:emacsql-sqlite-builtin")
+(declare-function emacsql-sqlite-module "ext:emacsql-sqlite-module")
+
+(defun org-roam-db--conn-fn ()
+  "Return the function for creating the database connection."
+  (cl-case org-roam-database-connector
+    (sqlite
+     (progn
+       (require 'emacsql-sqlite)
+       #'emacsql-sqlite))
+    (sqlite-builtin
+     (progn
+       (require 'emacsql-sqlite-builtin)
+       #'emacsql-sqlite-builtin))
+    (sqlite-module
+     (progn
+       (require 'emacsql-sqlite-module)
+       #'emacsql-sqlite-module))
+    (libsqlite3
+     (progn
+       (require 'emacsql-libsqlite3)
+       #'emacsql-libsqlite3))
+    (sqlite3
+     (progn
+       (require 'emacsql-sqlite3)
+       #'emacsql-sqlite3))))
+
 (defun org-roam-db ()
   "Entrypoint to the Org-roam sqlite database.
 Initializes and stores the database, and the database connection.
@@ -104,9 +186,12 @@ Performs a database upgrade when required."
                (emacsql-live-p (org-roam-db--get-connection)))
     (let ((init-db (not (file-exists-p org-roam-db-location))))
       (make-directory (file-name-directory org-roam-db-location) t)
-      (let ((conn (emacsql-sqlite org-roam-db-location)))
-        (set-process-query-on-exit-flag (emacsql-process conn) nil)
-        (puthash (expand-file-name org-roam-directory)
+      (let ((conn (funcall (org-roam-db--conn-fn) org-roam-db-location)))
+        (emacsql conn [:pragma (= foreign_keys ON)])
+        (when-let* ((process (emacsql-process conn))
+                    (_ (processp process)))
+          (set-process-query-on-exit-flag process nil))
+        (puthash (expand-file-name (file-name-as-directory org-roam-directory))
                  conn
                  org-roam-db--connection)
         (when init-db
@@ -145,6 +230,7 @@ The query is expected to be able to fail, in this situation, run HANDLER."
 (defconst org-roam-db--table-schemata
   '((files
      [(file :unique :primary-key)
+      title
       (hash :not-null)
       (atime :not-null)
       (mtime :not-null)])
@@ -168,6 +254,13 @@ The query is expected to be able to fail, in this situation, run HANDLER."
        alias]
       (:foreign-key [node-id] :references nodes [id] :on-delete :cascade)))
 
+    (citations
+     ([(node-id :not-null)
+       (cite-key :not-null)
+       (pos :not-null)
+       properties]
+      (:foreign-key [node-id] :references nodes [id] :on-delete :cascade)))
+
     (refs
      ([(node-id :not-null)
        (ref :not-null)
@@ -195,7 +288,6 @@ The query is expected to be able to fail, in this situation, run HANDLER."
 (defun org-roam-db--init (db)
   "Initialize database DB with the correct schema and user version."
   (emacsql-with-transaction db
-    (emacsql db "PRAGMA foreign_keys = ON")
     (pcase-dolist (`(,table ,schema) org-roam-db--table-schemata)
       (emacsql db [:create-table $i1 $S2] table schema))
     (pcase-dolist (`(,index-name ,table ,columns) org-roam-db--table-indices)
@@ -246,10 +338,22 @@ If FILE is nil, clear the current buffer."
                      file))
 
 ;;;; Updating tables
+
+(defun org-roam-db--file-title ()
+  "In current Org buffer, get the title.
+If there is no title, return the file name relative to
+`org-roam-directory'."
+  (org-link-display-format
+   (or (cadr (assoc "TITLE" (org-collect-keywords '("title"))))
+       (file-name-sans-extension (file-relative-name
+                                  (buffer-file-name (buffer-base-buffer))
+                                  org-roam-directory)))))
+
 (defun org-roam-db-insert-file ()
   "Update the files table for the current buffer.
 If UPDATE-P is non-nil, first remove the file in the database."
   (let* ((file (buffer-file-name))
+         (file-title (org-roam-db--file-title))
          (attr (file-attributes file))
          (atime (file-attribute-access-time attr))
          (mtime (file-attribute-modification-time attr))
@@ -257,7 +361,7 @@ If UPDATE-P is non-nil, first remove the file in the database."
     (org-roam-db-query
      [:insert :into files
       :values $v1]
-     (list (vector file hash atime mtime)))))
+     (list (vector file file-title hash atime mtime)))))
 
 (defun org-roam-db-get-scheduled-time ()
   "Return the scheduled time at point in ISO8601 format."
@@ -272,25 +376,57 @@ If UPDATE-P is non-nil, first remove the file in the database."
 (defun org-roam-db-node-p ()
   "Return t if headline at point is an Org-roam node, else return nil."
   (and (org-id-get)
-       (not (cdr (assoc "ROAM_EXCLUDE" (org-entry-properties))))
+       (not (org-entry-get (point) "ROAM_EXCLUDE"))
        (funcall org-roam-db-node-include-function)))
 
 (defun org-roam-db-map-nodes (fns)
   "Run FNS over all nodes in the current buffer."
-  (org-with-point-at 1
-    (org-map-entries
-     (lambda ()
-       (when (org-roam-db-node-p)
-         (dolist (fn fns)
-           (funcall fn)))))))
+  (org-map-region
+   (lambda ()
+     (when (org-roam-db-node-p)
+       (dolist (fn fns)
+         (funcall fn))))
+   (point-min) (point-max)))
 
 (defun org-roam-db-map-links (fns)
   "Run FNS over all links in the current buffer."
   (org-with-point-at 1
-    (org-element-map (org-element-parse-buffer) 'link
-      (lambda (link)
-        (dolist (fn fns)
-          (funcall fn link))))))
+    (while (re-search-forward org-link-any-re nil :no-error)
+      ;; `re-search-forward' let the cursor one character after the link, we need to go backward one char to
+      ;; make the point be on the link.
+      (backward-char)
+      (let* ((element (org-element-context))
+             (type (org-element-type element))
+             link bounds)
+        (cond
+         ;; Links correctly recognized by Org Mode
+         ((eq type 'link)
+          (setq link element))
+         ;; Links in property drawers and lines starting with #+. Recall that, as for Org Mode v9.4.4, the
+         ;; org-element-type of links within properties drawers is "node-property" and for lines starting with
+         ;; #+ is "keyword".
+         ((and (member type org-roam-db-extra-links-elements)
+               (not (member-ignore-case (org-element-property :key element)
+                                        (cdr (assoc type org-roam-db-extra-links-exclude-keys))))
+               (setq bounds (org-in-regexp org-link-any-re))
+               (setq link (buffer-substring-no-properties
+                           (car bounds)
+                           (cdr bounds))))
+          (with-temp-buffer
+            (delay-mode-hooks (org-mode))
+            (insert link)
+            (setq link (org-element-context)))))
+        (when link
+          (dolist (fn fns)
+            (funcall fn link)))))))
+
+(defun org-roam-db-map-citations (info fns)
+  "Run FNS over all citations in the current buffer.
+INFO is the org-element parsed buffer."
+  (org-element-map info 'citation-reference
+    (lambda (cite)
+      (dolist (fn fns)
+        (funcall fn cite)))))
 
 (defun org-roam-db-insert-file-node ()
   "Insert the file-level node into the Org-roam cache."
@@ -299,19 +435,14 @@ If UPDATE-P is non-nil, first remove the file in the database."
                (org-roam-db-node-p))
       (when-let ((id (org-id-get)))
         (let* ((file (buffer-file-name (buffer-base-buffer)))
-               (title (org-link-display-format
-                       (or (cadr (assoc "TITLE" (org-collect-keywords '("title"))
-                                        #'string-equal))
-                           (file-relative-name file org-roam-directory))))
+               (title (org-roam-db--file-title))
                (pos (point))
                (todo nil)
                (priority nil)
                (scheduled nil)
                (deadline nil)
                (level 0)
-               (aliases (org-entry-get (point) "ROAM_ALIASES"))
                (tags org-file-tags)
-               (refs (org-entry-get (point) "ROAM_REFS"))
                (properties (org-entry-properties))
                (olp nil))
           (org-roam-db-query!
@@ -330,29 +461,8 @@ If UPDATE-P is non-nil, first remove the file in the database."
              (mapcar (lambda (tag)
                        (vector id (substring-no-properties tag)))
                      tags)))
-          (when aliases
-            (org-roam-db-query
-             [:insert :into aliases
-              :values $v1]
-             (mapcar (lambda (alias)
-                       (vector id alias))
-                     (split-string-and-unquote aliases))))
-          (when refs
-            (setq refs (split-string-and-unquote refs))
-            (let (rows)
-              (dolist (ref refs)
-                (if (string-match org-link-plain-re ref)
-                    (progn
-                      (push (vector id (match-string 2 ref)
-                                    (match-string 1 ref)) rows))
-                  (lwarn '(org-roam) :warning
-                         "%s:%s\tInvalid ref %s, skipping..."
-                         (buffer-file-name) (point) ref)))
-              (when rows
-                (org-roam-db-query
-                 [:insert :into refs
-                  :values $v1]
-                 rows)))))))))
+          (org-roam-db-insert-aliases)
+          (org-roam-db-insert-refs))))))
 
 (cl-defun org-roam-db-insert-node-data ()
   "Insert node data for headline at point into the Org-roam cache."
@@ -386,13 +496,14 @@ If UPDATE-P is non-nil, first remove the file in the database."
 
 (defun org-roam-db-insert-aliases ()
   "Insert aliases for node at point into Org-roam cache."
-  (when-let ((node-id (org-id-get))
-             (aliases (org-entry-get (point) "ROAM_ALIASES")))
+  (when-let* ((node-id (org-id-get))
+              (aliases (org-entry-get (point) "ROAM_ALIASES"))
+              (aliases (split-string-and-unquote aliases)))
     (org-roam-db-query [:insert :into aliases
                         :values $v1]
                        (mapcar (lambda (alias)
                                  (vector node-id alias))
-                               (split-string-and-unquote aliases)))))
+                               aliases))))
 
 (defun org-roam-db-insert-tags ()
   "Insert tags for node at point into Org-roam cache."
@@ -411,14 +522,39 @@ If UPDATE-P is non-nil, first remove the file in the database."
     (let (rows)
       (dolist (ref refs)
         (save-match-data
-          (if (string-match org-link-plain-re ref)
-              (progn
-                (push (vector node-id (match-string 2 ref) (match-string 1 ref)) rows))
-            (lwarn '(org-roam) :warning
-                   "%s:%s\tInvalid ref %s, skipping..." (buffer-file-name) (point) ref))))
-      (org-roam-db-query [:insert :into refs
-                          :values $v1]
-                         rows))))
+          (cond (;; @citeKey
+                 (string-prefix-p "@" ref)
+                 (push (vector node-id (substring ref 1) "cite") rows))
+                (;; [cite:@citeKey]
+                 (string-prefix-p "[cite:" ref)
+                 (condition-case nil
+                     (let ((cite-obj (org-cite-parse-objects ref)))
+                       (org-element-map cite-obj 'citation-reference
+                         (lambda (cite)
+                           (let ((key (org-element-property :key cite)))
+                             (push (vector node-id key "cite") rows)))))
+                   (error
+                    (lwarn '(org-roam) :warning
+                           "%s:%s\tInvalid cite %s, skipping..." (buffer-file-name) (point) ref))))
+                (;; https://google.com, cite:citeKey
+                 ;; Note: we use string-match here because it matches any link: e.g. [[cite:abc][abc]]
+                 ;; But this form of matching is loose, and can accept invalid links e.g. [[cite:abc]
+                 (string-match org-link-plain-re ref)
+                 (let ((link-type (match-string 1 ref))
+                       (path (match-string 2 ref)))
+                   (if (and (boundp 'org-ref-cite-types)
+                            (or (assoc link-type org-ref-cite-types)
+                                (member link-type org-ref-cite-types)))
+                       (dolist (key (org-roam-org-ref-path-to-keys path))
+                         (push (vector node-id key link-type) rows))
+                     (push (vector node-id path link-type) rows))))
+                (t
+                 (lwarn '(org-roam) :warning
+                        "%s:%s\tInvalid ref %s, skipping..." (buffer-file-name) (point) ref)))))
+      (when rows
+        (org-roam-db-query [:insert :into refs
+                            :values $v1]
+                           rows)))))
 
 (defun org-roam-db-insert-link (link)
   "Insert link data for LINK at current point into the Org-roam cache."
@@ -426,24 +562,39 @@ If UPDATE-P is non-nil, first remove the file in the database."
     (goto-char (org-element-property :begin link))
     (let ((type (org-element-property :type link))
           (path (org-element-property :path link))
+          (source (org-roam-id-at-point))
           (properties (list :outline (ignore-errors
                                        ;; This can error if link is not under any headline
-                                       (org-get-outline-path 'with-self 'use-cache))))
-          (source (org-roam-id-at-point)))
+                                       (org-get-outline-path 'with-self 'use-cache)))))
       ;; For Org-ref links, we need to split the path into the cite keys
-      (when (and (boundp 'org-ref-cite-types)
-                 (fboundp 'org-ref-split-and-strip-string)
-                 (member type org-ref-cite-types))
-        (setq path (org-ref-split-and-strip-string path)))
-      (unless (listp path)
-        (setq path (list path)))
       (when (and source path)
+        (if (and (boundp 'org-ref-cite-types)
+                 (or (assoc type org-ref-cite-types)
+                     (member type org-ref-cite-types)))
+            (org-roam-db-query
+             [:insert :into citations
+              :values $v1]
+             (mapcar (lambda (k) (vector source k (point) properties))
+                     (org-roam-org-ref-path-to-keys path)))
+          (org-roam-db-query
+           [:insert :into links
+            :values $v1]
+           (vector (point) source path type properties)))))))
+
+(defun org-roam-db-insert-citation (citation)
+  "Insert data for CITATION at current point into the Org-roam cache."
+  (save-excursion
+    (goto-char (org-element-property :begin citation))
+    (let ((key (org-element-property :key citation))
+          (source (org-roam-id-at-point))
+          (properties (list :outline (ignore-errors
+                                       ;; This can error if link is not under any headline
+                                       (org-get-outline-path 'with-self 'use-cache)))))
+      (when (and source key)
         (org-roam-db-query
-         [:insert :into links
+         [:insert :into citations
           :values $v1]
-         (mapcar (lambda (p)
-                   (vector (point) source p type properties))
-                 path))))))
+         (vector source key (point) properties))))))
 
 ;;;; Fetching
 (defun org-roam-db--get-current-files ()
@@ -469,30 +620,47 @@ If UPDATE-P is non-nil, first remove the file in the database."
      (secure-hash 'sha1 (current-buffer)))))
 
 ;;;; Synchronization
-(defun org-roam-db-update-file (&optional file-path)
+(defun org-roam-db-update-file (&optional file-path no-require)
   "Update Org-roam cache for FILE-PATH.
+
 If the file does not exist anymore, remove it from the cache.
-If the file exists, update the cache with information."
+
+If the file exists, update the cache with information.
+
+If NO-REQUIRE, don't require optional libraries. Set NO-REQUIRE
+when the libraries are already required at some toplevel, e.g.
+in `org-roam-db-sync'."
   (setq file-path (or file-path (buffer-file-name (buffer-base-buffer))))
   (let ((content-hash (org-roam-db--file-hash file-path))
         (db-hash (caar (org-roam-db-query [:select hash :from files
-                                           :where (= file $s1)] file-path))))
+                                           :where (= file $s1)] file-path)))
+        info)
     (unless (string= content-hash db-hash)
+      (unless no-require
+        (org-roam-require '(org-ref oc)))
       (org-roam-with-file file-path nil
-        (save-excursion
-          (org-set-regexps-and-options 'tags-only)
-          (org-roam-db-clear-file)
-          (org-roam-db-insert-file)
-          (org-roam-db-insert-file-node)
-          (setq org-outline-path-cache nil)
-          (org-roam-db-map-nodes
-           (list #'org-roam-db-insert-node-data
-                 #'org-roam-db-insert-aliases
-                 #'org-roam-db-insert-tags
-                 #'org-roam-db-insert-refs))
-          (setq org-outline-path-cache nil)
-          (org-roam-db-map-links
-           (list #'org-roam-db-insert-link)))))))
+        (emacsql-with-transaction (org-roam-db)
+          (org-with-wide-buffer
+           (org-set-regexps-and-options 'tags-only)
+           (org-refresh-category-properties)
+           (org-roam-db-clear-file)
+           (org-roam-db-insert-file)
+           (org-roam-db-insert-file-node)
+           (setq org-outline-path-cache nil)
+           (org-roam-db-map-nodes
+            (list #'org-roam-db-insert-node-data
+                  #'org-roam-db-insert-aliases
+                  #'org-roam-db-insert-tags
+                  #'org-roam-db-insert-refs))
+           (setq org-outline-path-cache nil)
+           (setq info (org-element-parse-buffer))
+           (org-roam-db-map-links
+            (list #'org-roam-db-insert-link))
+           (when (fboundp 'org-cite-insert)
+             (require 'oc)             ;ensure feature is loaded
+             (org-roam-db-map-citations
+              info
+              (list #'org-roam-db-insert-citation)))))))))
 
 ;;;###autoload
 (defun org-roam-db-sync (&optional force)
@@ -502,6 +670,7 @@ If FORCE, force a rebuild of the cache from scratch."
   (org-roam-db--close) ;; Force a reconnect
   (when force (delete-file org-roam-db-location))
   (org-roam-db) ;; To initialize the database, no-op if already initialized
+  (org-roam-require '(org-ref oc))
   (let* ((gc-cons-threshold org-roam-db-gc-threshold)
          (org-agenda-files nil)
          (org-roam-files (org-roam-list-files))
@@ -514,18 +683,17 @@ If FORCE, force a rebuild of the cache from scratch."
           (push file modified-files)))
       (remhash file current-files))
     (emacsql-with-transaction (org-roam-db)
-      (if (fboundp 'dolist-with-progress-reporter)
-          (dolist-with-progress-reporter (file (hash-table-keys current-files))
-              "Clearing removed files..."
-            (org-roam-db-clear-file file))
-        (dolist (file (hash-table-keys current-files))
-          (org-roam-db-clear-file file)))
-      (if (fboundp 'dolist-with-progress-reporter)
-          (dolist-with-progress-reporter (file modified-files)
-              "Processing modified files..."
-            (org-roam-db-update-file file))
-        (dolist (file modified-files)
-          (org-roam-db-update-file file))))))
+      (org-roam-dolist-with-progress (file (hash-table-keys current-files))
+          "Clearing removed files..."
+        (org-roam-db-clear-file file))
+      (org-roam-dolist-with-progress (file modified-files)
+          "Processing modified files..."
+        (condition-case err
+            (org-roam-db-update-file file 'no-require)
+          (error
+           (org-roam-db-clear-file file)
+           (lwarn 'org-roam :error "Failed to process %s with error %s, skipping..."
+                  file (error-message-string err))))))))
 
 ;;;###autoload
 (define-minor-mode org-roam-db-autosync-mode

org-roam-id.el

@@ -0,0 +1,117 @@
+;;; org-roam-id.el --- ID-related utilities for Org-roam -*- lexical-binding: t; -*-
+
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
+
+;; Author: Jethro Kuan <jethrokuan95@gmail.com>
+;; URL: https://github.com/org-roam/org-roam
+;; Keywords: org-mode, roam, convenience
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (magit-section "3.0.0"))
+
+;; This file is NOT part of GNU Emacs.
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs; see the file COPYING.  If not, write to the
+;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
+
+;;; Commentary:
+;;
+;; This module provides ID-related facilities using the Org-roam database.
+;;
+;;; Code:
+(require 'org-id)
+
+(defun org-roam-id-at-point ()
+  "Return the ID at point, if any.
+Recursively traverses up the headline tree to find the
+first encapsulating ID."
+  (org-with-wide-buffer
+   (org-back-to-heading-or-point-min t)
+   (while (and (not (org-roam-db-node-p))
+               (not (bobp)))
+     (org-roam-up-heading-or-point-min))
+   (when (org-roam-db-node-p)
+     (org-id-get))))
+
+(defun org-roam-id-find (id &optional markerp)
+  "Return the location of the entry with the id ID using the Org-roam db.
+The return value is a cons cell (file-name . position), or nil
+if there is no entry with that ID.
+With optional argument MARKERP, return the position as a new marker."
+  (cond
+   ((symbolp id) (setq id (symbol-name id)))
+   ((numberp id) (setq id (number-to-string id))))
+  (let ((node (org-roam-populate (org-roam-node-create :id id))))
+    (when-let ((file (org-roam-node-file node)))
+      (if markerp
+          (unwind-protect
+              (let ((buffer (or (find-buffer-visiting file)
+                                (find-file-noselect file))))
+                (with-current-buffer buffer
+                  (move-marker (make-marker) (org-roam-node-point node) buffer))))
+        (cons (org-roam-node-file node)
+              (org-roam-node-point node))))))
+
+(defun org-roam-id-open (id _)
+  "Go to the entry with id ID.
+Like `org-id-open', but additionally uses the Org-roam database."
+  (org-mark-ring-push)
+  (let ((m (or (org-roam-id-find id 'marker)
+               (org-id-find id 'marker)))
+        cmd)
+    (unless m
+      (error "Cannot find entry with ID \"%s\"" id))
+    ;; Use a buffer-switching command in analogy to finding files
+    (setq cmd
+          (or
+           (cdr
+            (assq
+             (cdr (assq 'file org-link-frame-setup))
+             '((find-file . switch-to-buffer)
+               (find-file-other-window . switch-to-buffer-other-window)
+               (find-file-other-frame . switch-to-buffer-other-frame))))
+           'switch-to-buffer-other-window))
+    (if (not (equal (current-buffer) (marker-buffer m)))
+        (funcall cmd (marker-buffer m)))
+    (goto-char m)
+    (move-marker m nil)
+    (org-show-context)))
+
+(org-link-set-parameters "id" :follow #'org-roam-id-open)
+
+;;;###autoload
+(defun org-roam-update-org-id-locations (&rest directories)
+  "Scan Org-roam files to update `org-id' related state.
+This is like `org-id-update-id-locations', but will automatically
+use the currently bound `org-directory' and `org-roam-directory'
+along with DIRECTORIES (if any), where the lookup for files in
+these directories will be always recursive.
+
+Note: Org-roam doesn't have hard dependency on
+`org-id-locations-file' to lookup IDs for nodes that are stored
+in the database, but it still tries to properly integrates with
+`org-id'. This allows the user to cross-reference IDs outside of
+the current `org-roam-directory', and also link with \"id:\"
+links to headings/files within the current `org-roam-directory'
+that are excluded from identification in Org-roam as
+`org-roam-node's, e.g. with \"ROAM_EXCLUDE\" property."
+  (interactive)
+  (cl-loop for dir in (cons org-roam-directory directories)
+           for org-roam-directory = dir
+           nconc (org-roam-list-files) into files
+           finally (org-id-update-id-locations files org-roam-verbose)))
+
+(provide 'org-roam-id)
+
+;;; org-roam-id.el ends here

org-roam-log.el

@@ -0,0 +1,53 @@
+;;; org-roam-log.el --- Integrations with Org-log -*- coding: utf-8; lexical-binding: t; -*-
+
+;; Copyright © 2022-2022 Jethro Kuan <jethrokuan95@gmail.com>
+
+;; Author: Jethro Kuan <jethrokuan95@gmail.com>
+;; URL: https://github.com/org-roam/org-roam
+;; Keywords: org-mode, roam, convenience
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
+
+;; This file is NOT part of GNU Emacs.
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3, or (at your option)
+;; any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs; see the file COPYING.  If not, write to the
+;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
+
+;;; Commentary:
+;;
+;; This module provides integrations with Org-log.
+;;
+;;; Code:
+(require 'org-roam)
+
+(defcustom org-roam-log-setup-hook nil
+  "Hook run when a log for an Org-roam file is setup."
+  :group 'org-roam
+  :type 'hook)
+
+(defun org-roam-log-p ()
+  "Return t if the log buffer is for an Org-roam file, nil otherwise."
+  (and org-log-note-marker
+       (org-roam-file-p (buffer-file-name (marker-buffer org-log-note-marker)))))
+
+(defun org-roam-log--setup ()
+  "Run hooks in `org-roam-log-setup-hook'."
+  (run-hooks 'org-roam-log-setup-hook))
+
+(add-hook 'org-roam-log-setup-hook #'org-roam--register-completion-functions-h)
+(add-hook 'org-log-buffer-setup-hook #'org-roam-log--setup)
+
+(provide 'org-roam-log)
+;;; org-roam-log.el ends here

org-roam-migrate.el

@@ -1,12 +1,12 @@
 ;;; org-roam-migrate.el --- Migration utilities from v1 to v2 -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -34,48 +34,6 @@
 ;;; Code:
 (require 'org-roam)
 
-;;; v1 breaking warning
-(defvar org-roam-v2-ack nil
-  "When set to t, won't display the annoying warning message about the upgrade.
-Need to be set before the package is loaded, otherwise won't take
-any affect.")
-
-(unless org-roam-v2-ack
-  (lwarn 'org-roam :error "
-------------------------------------
-WARNING: You're now on Org-roam v2!
-------------------------------------
-
-You may have arrived here from a package upgrade. Please read the
-wiki entry at
-%s
-for an overview of the major changes.
-
-Notes taken in v1 are incompatible with v2, but you can upgrade
-them to the v2 format via a simple command. To migrate your
-notes, first make sure you're on at least Org 9.4 (check with
-C-h v org-version) and set your org-roam-directory to your notes:
-
-(setq org-roam-directory \"path/to/org/files\")
-
-then, run:
-
-  M-x org-roam-migrate-wizard
-
-If you wish to stay on v1, v1 is unfortunately not distributed on
-MELPA. See org-roam/org-roam-v1 on GitHub on how to install v1.
-
-If you've gone through the migration steps (if necessary), and
-know what you're doing set `org-roam-v2-ack' to `t' to disable
-this warning. You can do so by adding:
-
-(setq org-roam-v2-ack t)
-
-To your init file.
-
-"
-         "https://github.com/org-roam/org-roam/wiki/Hitchhiker's-Rough-Guide-to-Org-roam-V2"))
-
 ;;; Migration wizard (v1 -> v2)
 ;;;###autoload
 (defun org-roam-migrate-wizard ()

org-roam-mode.el

@@ -1,12 +1,12 @@
 ;;; org-roam-mode.el --- Major mode for special Org-roam buffers -*- lexical-binding: t -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -39,12 +39,50 @@
 (defvar org-ref-buffer-hacked)
 
 ;;; Options
-(defcustom org-roam-mode-section-functions (list #'org-roam-backlinks-section
-                                                 #'org-roam-reflinks-section)
-  "Functions that insert sections in the `org-roam-mode' based buffers.
-Each function is called with one argument, which is an
-`org-roam-node' for which the buffer will be constructed for.
-Normally this node is `org-roam-buffer-current-node'."
+(defcustom org-roam-mode-sections (list #'org-roam-backlinks-section
+                                        #'org-roam-reflinks-section)
+  "A list of sections for the `org-roam-mode' based buffers.
+Each section is a function that is passed the an `org-roam-node'
+for which the section will be constructed for as the first
+argument. Normally this node is `org-roam-buffer-current-node'.
+The function may also accept other optional arguments. Each item
+in the list is either:
+
+1. A function, which is called only with the `org-roam-node' as the argument
+2. A list, containing the function and the optional arguments.
+
+For example, one can add
+
+    (org-roam-backlinks-section :unique t)
+
+to the list to pass :unique t to the section-rendering function."
+  :group 'org-roam
+  :type `(repeat (choice (symbol :tag "Function")
+                         (list :tag "Function with arguments"
+                               (symbol :tag "Function")
+                               (repeat :tag "Arguments" :inline t (sexp :tag "Arg"))))))
+
+(defcustom org-roam-buffer-postrender-functions (list)
+  "Functions to run after the Org-roam buffer is rendered.
+Each function accepts no arguments, and is run with the Org-roam
+buffer as the current buffer."
+  :group 'org-roam
+  :type 'hook)
+
+(defcustom org-roam-preview-function #'org-roam-preview-default-function
+  "The preview function to use to populate the Org-roam buffer.
+
+The function takes no arguments, but the point is temporarily set
+to the exact location of the backlink."
+  :group 'org-roam
+  :type 'function)
+
+(defcustom org-roam-preview-postprocess-functions (list #'org-roam-strip-comments)
+  "A list of functions to postprocess the preview content.
+
+Each function takes a single argument, the string for the preview
+content, and returns the post-processed string. The functions are
+applied in order of appearance in the list."
   :group 'org-roam
   :type 'hook)
 
@@ -144,7 +182,7 @@ This mode is used by special Org-roam buffers, such as persistent
 `org-roam-buffer' and dedicated Org-roam buffers
 \(`org-roam-buffer-display-dedicated'), which render the
 information in a section-like manner (see
-`org-roam-mode-section-functions'), with which the user can
+`org-roam-mode-sections'), with which the user can
 interact with."
   :group 'org-roam
   (face-remap-add-relative 'header-line 'org-roam-header-line))
@@ -160,7 +198,8 @@ value shows the current node in the persistent `org-roam-buffer'.")
 
 (defvar org-roam-buffer-current-directory nil
   "The `org-roam-directory' value of `org-roam-buffer-current-node'.
-Set both, locally and globally in the same way as `org-roam-buffer-current-node'.")
+Set both, locally and globally in the same way as
+`org-roam-buffer-current-node'.")
 
 (put 'org-roam-buffer-current-directory 'permanent-local t)
 
@@ -203,7 +242,15 @@ buffer."
      (org-roam-node-title org-roam-buffer-current-node))
     (magit-insert-section (org-roam)
       (magit-insert-heading)
-      (run-hook-with-args 'org-roam-mode-section-functions org-roam-buffer-current-node))
+      (dolist (section org-roam-mode-sections)
+        (pcase section
+          ((pred functionp)
+           (funcall section org-roam-buffer-current-node))
+          (`(,fn . ,args)
+           (apply fn (cons org-roam-buffer-current-node args)))
+          (_
+           (user-error "Invalid `org-roam-mode-sections' specification")))))
+    (run-hooks 'org-roam-buffer-postrender-functions)
     (goto-char 0)))
 
 (defun org-roam-buffer-set-header-line-format (string)
@@ -263,7 +310,7 @@ To toggle its display use `org-roam-buffer-toggle' command.")
   (pcase (org-roam-buffer--visibility)
     ('visible
      (progn
-       (delete-window (get-buffer-window org-roam-buffer))
+       (quit-window nil (get-buffer-window org-roam-buffer))
        (remove-hook 'post-command-hook #'org-roam-buffer--redisplay-h)))
     ((or 'exists 'none)
      (progn
@@ -397,87 +444,29 @@ In interactive calls OTHER-WINDOW is set with
     (when (org-invisible-p) (org-show-context))
     buf))
 
-(defun org-roam-preview-get-contents (file point)
-  "Get preview content for FILE at POINT."
+(defun org-roam-preview-default-function ()
+  "Return the preview content at point.
+
+This function returns the all contents under the current
+headline, up to the next headline."
+  (let ((beg (save-excursion
+               (org-roam-end-of-meta-data t)
+               (point)))
+        (end (save-excursion
+               (org-next-visible-heading 1)
+               (point))))
+    (string-trim (buffer-substring-no-properties beg end))))
+
+(defun org-roam-preview-get-contents (file pt)
+  "Get preview content for FILE at PT."
   (save-excursion
     (org-roam-with-temp-buffer file
-      (goto-char point)
-      (let ((elem (org-element-at-point)))
-        ;; We want the parent element always
-        (while (org-element-property :parent elem)
-          (setq elem (org-element-property :parent elem)))
-        (pcase (car elem)
-          ('headline                    ; show subtree
-           (org-roam-preview-get-entry-text (point-marker) most-positive-fixnum))
-          (_
-           (let ((begin (org-element-property :begin elem))
-                 (end (org-element-property :end elem)))
-             (or (string-trim (buffer-substring-no-properties begin end))
-                 (org-element-property :raw-value elem)))))))))
-
-(defun org-roam-preview-get-entry-text (marker n-lines &optional indent)
-  "Extract entry text from MARKER, at most N-LINES lines.
-This will ignore drawers etc, just get the text.
-If INDENT is given, prefix every line with this string."
-  (let (txt ind)
-    (save-excursion
-      (with-current-buffer (marker-buffer marker)
-        (if (not (derived-mode-p 'org-mode))
-            (setq txt "")
-          (org-with-wide-buffer
-           (goto-char marker)
-           (end-of-line 1)
-           (setq txt (buffer-substring
-                      (min (1+ (point)) (point-max))
-                      (progn (outline-next-heading) (point))))
-           (with-temp-buffer
-             (insert txt)
-             (goto-char (point-min))
-             (while (org-activate-links (point-max))
-               (goto-char (match-end 0)))
-             (goto-char (point-min))
-             (while (re-search-forward org-link-bracket-re (point-max) t)
-               (set-text-properties (match-beginning 0) (match-end 0)
-                                    nil))
-             (goto-char (point-min))
-             (while (re-search-forward org-drawer-regexp nil t)
-               (delete-region
-                (match-beginning 0)
-                (progn (re-search-forward
-                        "^[ \t]*:END:.*\n?" nil 'move)
-                       (point))))
-             (goto-char (point-min))
-             (goto-char (point-max))
-             (skip-chars-backward " \t\n")
-             (when (looking-at "[ \t\n]+\\'") (replace-match ""))
-
-             ;; find and remove min common indentation
-             (goto-char (point-min))
-             (untabify (point-min) (point-max))
-             (setq ind (current-indentation))
-             (while (not (eobp))
-               (unless (looking-at "[ \t]*$")
-                 (setq ind (min ind (current-indentation))))
-               (beginning-of-line 2))
-             (goto-char (point-min))
-             (while (not (eobp))
-               (unless (looking-at "[ \t]*$")
-                 (move-to-column ind)
-                 (delete-region (point-at-bol) (point)))
-               (beginning-of-line 2))
-             (goto-char (point-min))
-             (when indent
-               (while (and (not (eobp)) (re-search-forward "^" nil t))
-                 (replace-match indent t t)))
-             (goto-char (point-min))
-             (while (looking-at "[ \t]*\n") (replace-match ""))
-             (goto-char (point-max))
-             (when (> (org-current-line)
-                      n-lines)
-               (org-goto-line (1+ n-lines))
-               (backward-char 1))
-             (setq txt (buffer-substring (point-min) (point))))))))
-    txt))
+      (org-with-wide-buffer
+       (goto-char pt)
+       (let ((s (funcall org-roam-preview-function)))
+         (dolist (fn org-roam-preview-postprocess-functions)
+           (setq s (funcall fn s)))
+         s)))))
 
 ;;;; Backlinks
 (cl-defstruct (org-roam-backlink (:constructor org-roam-backlink-create)
@@ -493,14 +482,23 @@ If INDENT is given, prefix every line with this string."
         (org-roam-populate (org-roam-backlink-target-node backlink)))
   backlink)
 
-(defun org-roam-backlinks-get (node)
-  "Return the backlinks for NODE."
-  (let ((backlinks (org-roam-db-query
-                    [:select [source dest pos properties]
-                     :from links
-                     :where (= dest $s1)
-                     :and (= type "id")]
-                    (org-roam-node-id node))))
+(cl-defun org-roam-backlinks-get (node &key unique)
+  "Return the backlinks for NODE.
+
+ When UNIQUE is nil, show all positions where references are found.
+ When UNIQUE is t, limit to unique sources."
+  (let* ((sql (if unique
+                  [:select :distinct [source dest pos properties]
+                   :from links
+                   :where (= dest $s1)
+                   :and (= type "id")
+                   :group :by source
+                   :having (funcall min pos)]
+                [:select [source dest pos properties]
+                 :from links
+                 :where (= dest $s1)
+                 :and (= type "id")]))
+         (backlinks (org-roam-db-query sql (org-roam-node-id node))))
     (cl-loop for backlink in backlinks
              collect (pcase-let ((`(,source-id ,dest-id ,pos ,properties) backlink))
                        (org-roam-populate
@@ -516,9 +514,12 @@ Sorts by title."
   (string< (org-roam-node-title (org-roam-backlink-source-node a))
            (org-roam-node-title (org-roam-backlink-source-node b))))
 
-(defun org-roam-backlinks-section (node)
-  "The backlinks section for NODE."
-  (when-let ((backlinks (seq-sort #'org-roam-backlinks-sort (org-roam-backlinks-get node))))
+(cl-defun org-roam-backlinks-section (node &key (unique nil))
+  "The backlinks section for NODE.
+
+When UNIQUE is nil, show all positions where references are found.
+When UNIQUE is t, limit to unique sources."
+  (when-let ((backlinks (seq-sort #'org-roam-backlinks-sort (org-roam-backlinks-get node :unique unique))))
     (magit-insert-section (org-roam-backlinks)
       (magit-insert-heading "Backlinks:")
       (dolist (backlink backlinks)
@@ -546,7 +547,14 @@ Sorts by title."
                                   :from refs
                                   :left-join links
                                   :where (= refs:node-id $s1)
-                                  :and (= links:dest refs:ref)]
+                                  :and (= links:dest refs:ref)
+                                  :union
+                                  :select :distinct [refs:ref citations:node-id
+                                                     citations:pos citations:properties]
+                                  :from refs
+                                  :left-join citations
+                                  :where (= refs:node-id $s1)
+                                  :and (= citations:cite-key refs:ref)]
                                  (org-roam-node-id node)))
         links)
     (pcase-dolist (`(,ref ,source-id ,pos ,properties) refs)
@@ -566,16 +574,16 @@ Sorts by title."
 
 (defun org-roam-reflinks-section (node)
   "The reflinks section for NODE."
-  (when (org-roam-node-refs node)
-    (let* ((reflinks (seq-sort #'org-roam-reflinks-sort (org-roam-reflinks-get node))))
-      (magit-insert-section (org-roam-reflinks)
-        (magit-insert-heading "Reflinks:")
-        (dolist (reflink reflinks)
-          (org-roam-node-insert-section
-           :source-node (org-roam-reflink-source-node reflink)
-           :point (org-roam-reflink-point reflink)
-           :properties (org-roam-reflink-properties reflink)))
-        (insert ?\n)))))
+  (when-let ((refs (org-roam-node-refs node))
+             (reflinks (seq-sort #'org-roam-reflinks-sort (org-roam-reflinks-get node))))
+    (magit-insert-section (org-roam-reflinks)
+      (magit-insert-heading "Reflinks:")
+      (dolist (reflink reflinks)
+        (org-roam-node-insert-section
+         :source-node (org-roam-reflink-source-node reflink)
+         :point (org-roam-reflink-point reflink)
+         :properties (org-roam-reflink-properties reflink)))
+      (insert ?\n))))
 
 ;;;; Grep
 (defvar org-roam-grep-map
@@ -649,7 +657,7 @@ References from FILE are excluded."
                                 (shell-command-to-string "rg --pcre2-version"))))
     (let* ((titles (cons (org-roam-node-title node)
                          (org-roam-node-aliases node)))
-           (rg-command (concat "rg -o --vimgrep -P -i "
+           (rg-command (concat "rg -L -o --vimgrep -P -i "
                                (mapconcat (lambda (glob) (concat "-g " glob))
                                           (org-roam--list-files-search-globs org-roam-file-extensions)
                                           " ")
@@ -670,14 +678,14 @@ References from FILE are excluded."
                     col (string-to-number (match-string 3 line))
                     match (match-string 4 line))
               (when (and match
-                         (not (f-equal-p (org-roam-node-file node) f))
+                         (not (file-equal-p (org-roam-node-file node) f))
                          (member (downcase match) (mapcar #'downcase titles)))
                 (magit-insert-section section (org-roam-grep-section)
                   (oset section file f)
                   (oset section row row)
                   (oset section col col)
                   (insert (propertize (format "%s:%s:%s"
-                                              (truncate-string-to-width (file-name-base f) 15 nil nil "...")
+                                              (truncate-string-to-width (file-name-base f) 15 nil nil t)
                                               row col) 'font-lock-face 'org-roam-dim)
                           " "
                           (org-roam-fontify-like-in-org-mode

org-roam-node.el

@@ -1,12 +1,12 @@
 ;;; org-roam-node.el --- Interfacing and interacting with nodes -*- lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -36,8 +36,7 @@
 
 ;;; Options
 ;;;; Completing-read
-(defcustom org-roam-node-display-template
-  "${title:*} ${tags:10}"
+(defcustom org-roam-node-display-template "${title}"
   "Configures display formatting for Org-roam node.
 Patterns of form \"${field-name:length}\" are interpolated based
 on the current node.
@@ -60,9 +59,13 @@ field. If it's not specified, the field will be inserted as is,
 i.e. it won't be aligned nor trimmed. If it's an integer, the
 field will be aligned accordingly and all the exceeding
 characters will be trimmed out. If it's \"*\", the field will use
-as many characters as possible and will be aligned accordingly."
+as many characters as possible and will be aligned accordingly.
+
+A closure can also be assigned to this variable in which case the
+closure is evaluated and the return value is used as the
+template. The closure must evaluate to a valid template string."
   :group 'org-roam
-  :type  'string)
+  :type  '(string function))
 
 (defcustom org-roam-node-annotation-function #'org-roam-node-read--annotation
   "This function used to attach annotations for `org-roam-node-read'.
@@ -72,10 +75,22 @@ It takes a single argument NODE, which is an `org-roam-node' construct."
 
 (defcustom org-roam-node-default-sort 'file-mtime
   "Default sort order for Org-roam node completions."
-  :type '(choice (const :tag "file-mtime" file-mtime)
-                 (const :tag "file-atime" file-atime))
+  :type '(choice
+          (const :tag "none" nil)
+          (const :tag "file-mtime" file-mtime)
+          (const :tag "file-atime" file-atime))
   :group 'org-roam)
 
+(defcustom org-roam-node-formatter nil
+  "The link description for node insertion.
+If a function is provided, the function should take a single
+argument, an `org-roam-node', and return a string.
+
+If a string is provided, it is a template string expanded by
+`org-roam-node--format-entry'."
+  :group 'org-roam
+  :type '(string function))
+
 (defcustom org-roam-node-template-prefixes
   '(("tags" . "#")
     ("todo" . "t:"))
@@ -93,7 +108,9 @@ has the entry (\"tags\" . \"#\"), these will appear as
 
 (defcustom org-roam-ref-annotation-function #'org-roam-ref-read--annotation
   "This function used to attach annotations for `org-roam-ref-read'.
-It takes a single argument REF, which is a propertized string.")
+It takes a single argument REF, which is a propertized string."
+  :group 'org-roam
+  :type  '(function))
 
 ;;;; Completion-at-point
 (defcustom org-roam-completion-everywhere nil
@@ -114,18 +131,36 @@ It takes a single argument REF, which is a propertized string.")
   :type 'boolean)
 
 (defcustom org-roam-extract-new-file-path "%<%Y%m%d%H%M%S>-${slug}.org"
-  "The file path to use when a node is extracted to its own file."
+  "The file path template to use when a node is extracted to its own file.
+This path is relative to `org-roam-directory'."
   :group 'org-roam
   :type 'string)
 
+(defvar org-roam-node-history nil
+  "Minibuffer history of nodes.")
+
+(defvar org-roam-ref-history nil
+  "Minibuffer history of refs.")
+
 ;;; Definition
 (cl-defstruct (org-roam-node (:constructor org-roam-node-create)
                              (:copier nil))
   "A heading or top level file with an assigned ID property."
-  file file-hash file-atime file-mtime
+  file file-title file-hash file-atime file-mtime
   id level point todo priority scheduled deadline title properties olp
   tags aliases refs)
 
+;; Shim `string-glyph-compose' and `string-glyph-decompose' for Emacs versions that do not have it.
+;; The functions were introduced in emacs commit 3f096eb3405b2fce7c35366eb2dcf025dda55783 and the
+;; (original) functions behind them aren't autoloaded anymore.
+(dolist (sym.replace
+         '((string-glyph-compose . ucs-normalize-NFC-string)
+           (string-glyph-decompose . ucs-normalize-NFD-string)))
+  (let ((emacs-29-symbol (car sym.replace))
+        (previous-implementation (cdr sym.replace)))
+    (unless (fboundp emacs-29-symbol)
+      (defalias emacs-29-symbol previous-implementation))))
+
 (cl-defmethod org-roam-node-slug ((node org-roam-node))
   "Return the slug of NODE."
   (let ((title (org-roam-node-title node))
@@ -140,6 +175,7 @@ It takes a single argument REF, which is a propertized string.")
                            776 ; U+0308 COMBINING DIAERESIS
                            777 ; U+0309 COMBINING HOOK ABOVE
                            778 ; U+030A COMBINING RING ABOVE
+                           779 ; U+030B COMBINING DOUBLE ACUTE ACCENT
                            780 ; U+030C COMBINING CARON
                            795 ; U+031B COMBINING HORN
                            803 ; U+0323 COMBINING DOT BELOW
@@ -151,14 +187,12 @@ It takes a single argument REF, which is a propertized string.")
                            816 ; U+0330 COMBINING TILDE BELOW
                            817 ; U+0331 COMBINING MACRON BELOW
                            )))
-    (cl-flet* ((nonspacing-mark-p (char)
-                                  (memq char slug-trim-chars))
-               (strip-nonspacing-marks (s)
-                                       (ucs-normalize-NFC-string
-                                        (apply #'string (seq-remove #'nonspacing-mark-p
-                                                                    (ucs-normalize-NFD-string s)))))
-               (cl-replace (title pair)
-                           (replace-regexp-in-string (car pair) (cdr pair) title)))
+    (cl-flet* ((nonspacing-mark-p (char) (memq char slug-trim-chars))
+               (strip-nonspacing-marks (s) (string-glyph-compose
+                                            (apply #'string
+                                                   (seq-remove #'nonspacing-mark-p
+                                                               (string-glyph-decompose s)))))
+               (cl-replace (title pair) (replace-regexp-in-string (car pair) (cdr pair) title)))
       (let* ((pairs `(("[^[:alnum:][:digit:]]" . "_") ;; convert anything not alphanumeric
                       ("__*" . "_")                   ;; remove sequential underscores
                       ("^_" . "")                     ;; remove starting underscore
@@ -166,6 +200,16 @@ It takes a single argument REF, which is a propertized string.")
              (slug (-reduce-from #'cl-replace (strip-nonspacing-marks title) pairs)))
         (downcase slug)))))
 
+(cl-defmethod org-roam-node-formatted ((node org-roam-node))
+  "Return a formatted string for NODE."
+  (pcase org-roam-node-formatter
+    ((pred functionp)
+     (funcall org-roam-node-formatter node))
+    ((pred stringp)
+     (org-roam-node--format-entry (org-roam-node--process-display-format org-roam-node-formatter) node))
+    (_
+     (org-roam-node-title node))))
+
 ;;; Nodes
 ;;;; Getters
 (defun org-roam-node-at-point (&optional assert)
@@ -223,9 +267,15 @@ Throw an error if multiple choices exist."
   "Return an `org-roam-node' from REF reference.
 Return nil if there's no node with such REF."
   (save-match-data
-    (when (string-match org-link-plain-re ref)
-      (let ((type (match-string 1 ref))
-            (path (match-string 2 ref)))
+    (let (type path)
+      (cond
+       ((string-match org-link-plain-re ref)
+        (setq type (match-string 1 ref)
+              path (match-string 2 ref)))
+       ((string-prefix-p "@" ref)
+        (setq type "cite"
+              path (substring ref 1))))
+      (when (and type path)
         (when-let ((id (caar (org-roam-db-query
                               [:select [nodes:id]
                                :from refs
@@ -249,10 +299,10 @@ nodes."
                                                  :limit 1]
                                                 (org-roam-node-id node)))))
     (pcase-let* ((`(,file ,level ,pos ,todo ,priority ,scheduled ,deadline ,title ,properties ,olp) node-info)
-                 (`(,atime ,mtime) (car (org-roam-db-query [:select [atime mtime]
-                                                            :from files
-                                                            :where (= file $s1)]
-                                                           file)))
+                 (`(,atime ,mtime ,file-title) (car (org-roam-db-query [:select [atime mtime title]
+                                                                        :from files
+                                                                        :where (= file $s1)]
+                                                                       file)))
                  (tag-info (mapcar #'car (org-roam-db-query [:select [tag] :from tags
                                                              :where (= node-id $s1)]
                                                             (org-roam-node-id node))))
@@ -263,6 +313,7 @@ nodes."
                                                               :where (= node-id $s1)]
                                                              (org-roam-node-id node)))))
       (setf (org-roam-node-file node) file
+            (org-roam-node-file-title node) file-title
             (org-roam-node-file-atime node) atime
             (org-roam-node-file-mtime node) mtime
             (org-roam-node-level node) level
@@ -285,6 +336,7 @@ nodes."
                "SELECT
   id,
   file,
+  filetitle,
   \"level\",
   todo,
   pos,
@@ -304,6 +356,7 @@ FROM
   SELECT
     id,
     file,
+    filetitle,
     \"level\",
     todo,
     pos,
@@ -334,6 +387,7 @@ FROM
       nodes.olp as olp,
       files.atime as atime,
       files.mtime as mtime,
+      files.title as filetitle,
       tags.tag as tags,
       aliases.alias as aliases,
       '(' || group_concat(RTRIM (refs.\"type\", '\"') || ':' || LTRIM(refs.ref, '\"'), ' ') || ')' as refs
@@ -346,13 +400,14 @@ FROM
   GROUP BY id, tags )
 GROUP BY id")))
     (cl-loop for row in rows
-             append (pcase-let* ((`(,id ,file ,level ,todo ,pos ,priority ,scheduled ,deadline
+             append (pcase-let* ((`(,id ,file ,file-title ,level ,todo ,pos ,priority ,scheduled ,deadline
                                         ,title ,properties ,olp ,atime ,mtime ,tags ,aliases ,refs)
                                   row)
                                  (all-titles (cons title aliases)))
                       (mapcar (lambda (temp-title)
                                 (org-roam-node-create :id id
                                                       :file file
+                                                      :file-title file-title
                                                       :file-atime atime
                                                       :file-mtime mtime
                                                       :level level
@@ -362,6 +417,7 @@ GROUP BY id")))
                                                       :scheduled scheduled
                                                       :deadline deadline
                                                       :title temp-title
+                                                      :aliases aliases
                                                       :properties properties
                                                       :olp olp
                                                       :tags tags
@@ -369,19 +425,41 @@ GROUP BY id")))
                               all-titles)))))
 
 ;;;; Finders
-(defun org-roam-node-find-noselect (node &optional force)
-  "Navigate to the point for NODE, and return the buffer.
-If NODE is already visited, this won't automatically move the
-point to the beginning of the NODE, unless FORCE is non-nil."
-  (unless (org-roam-node-file node)
-    (user-error "Node does not have corresponding file"))
-  (let ((buf (find-file-noselect (org-roam-node-file node))))
-    (with-current-buffer buf
-      (when (or force
-                (not (equal (org-roam-node-id node)
-                            (org-roam-id-at-point))))
-        (goto-char (org-roam-node-point node))))
-    buf))
+(defun org-roam-node-marker (node)
+  "Get the marker for NODE."
+  (unwind-protect
+      (let* ((file (org-roam-node-file node))
+             (buffer (or (find-buffer-visiting file)
+                         (find-file-noselect file))))
+        (with-current-buffer buffer
+          (move-marker (make-marker) (org-roam-node-point node) buffer)))))
+
+(defun org-roam-node-open (node &optional cmd force)
+  "Go to the node NODE.
+CMD is the command used to display the buffer. If not provided,
+`org-link-frame-setup' is respected. Assumes that the node is
+fully populated, with file and point. If NODE is already visited,
+this won't automatically move the point to the beginning of the
+NODE, unless FORCE is non-nil."
+  (interactive (list (org-roam-node-at-point) current-prefix-arg))
+  (org-mark-ring-push)
+  (let ((m (org-roam-node-marker node))
+        (cmd (or cmd
+                 (cdr
+                  (assq
+                   (cdr (assq 'file org-link-frame-setup))
+                   '((find-file . switch-to-buffer)
+                     (find-file-other-window . switch-to-buffer-other-window)
+                     (find-file-other-frame . switch-to-buffer-other-frame))))
+                 'switch-to-buffer-other-window)))
+    (if (not (equal (current-buffer) (marker-buffer m)))
+        (funcall cmd (marker-buffer m)))
+    (when (or force
+              (not (equal (org-roam-node-id node)
+                          (org-roam-id-at-point))))
+      (goto-char m))
+    (move-marker m nil))
+  (org-show-context))
 
 (defun org-roam-node-visit (node &optional other-window force)
   "From the current buffer, visit NODE. Return the visited buffer.
@@ -393,13 +471,10 @@ If NODE is already visited, this won't automatically move the
 point to the beginning of the NODE, unless FORCE is non-nil. In
 interactive calls FORCE always set to t."
   (interactive (list (org-roam-node-at-point t) current-prefix-arg t))
-  (let ((buf (org-roam-node-find-noselect node 'force))
-        (display-buffer-fn (if other-window
+  (org-roam-node-open node (if other-window
                                #'switch-to-buffer-other-window
-                             #'pop-to-buffer-same-window)))
-    (funcall display-buffer-fn buf)
-    (when (org-invisible-p) (org-show-context))
-    buf))
+                             #'pop-to-buffer-same-window)
+                      force))
 
 ;;;###autoload
 (cl-defun org-roam-node-find (&optional other-window initial-input filter-fn &key templates)
@@ -420,70 +495,89 @@ The TEMPLATES, if provided, override the list of capture templates (see
        :props '(:finalize find-file)))))
 
 ;;;###autoload
-(defun org-roam-node-random (&optional other-window)
+(defun org-roam-node-random (&optional other-window filter-fn)
   "Find and open a random Org-roam node.
 With prefix argument OTHER-WINDOW, visit the node in another
-window instead."
+window instead.
+FILTER-FN is a function to filter out nodes: it takes an `org-roam-node',
+and when nil is returned the node will be filtered out."
   (interactive current-prefix-arg)
-  (let ((random-row (seq-random-elt (org-roam-db-query [:select [id file pos] :from nodes]))))
-    (org-roam-node-visit (org-roam-node-create :id (nth 0 random-row)
-                                               :file (nth 1 random-row)
-                                               :point (nth 2 random-row))
-                         other-window)))
+  (org-roam-node-visit
+   (cdr (seq-random-elt (org-roam-node-read--completions filter-fn)))
+   other-window))
 
 ;;;; Completing-read interface
-(defun org-roam-node-read (&optional initial-input filter-fn sort-fn require-match)
+(defun org-roam-node-read (&optional initial-input filter-fn sort-fn require-match prompt)
   "Read and return an `org-roam-node'.
 INITIAL-INPUT is the initial minibuffer prompt value.
 FILTER-FN is a function to filter out nodes: it takes an `org-roam-node',
 and when nil is returned the node will be filtered out.
 SORT-FN is a function to sort nodes. See `org-roam-node-read-sort-by-file-mtime'
 for an example sort function.
-If REQUIRE-MATCH, the minibuffer prompt will require a match."
-  (let* ((nodes (org-roam-node-read--completions))
-         (nodes (cl-remove-if-not (lambda (n)
-                                    (if filter-fn (funcall filter-fn (cdr n)) t)) nodes))
+If REQUIRE-MATCH, the minibuffer prompt will require a match.
+PROMPT is a string to show at the beginning of the mini-buffer, defaulting to \"Node: \""
+  (let* ((nodes (org-roam-node-read--completions filter-fn sort-fn))
+         (prompt (or prompt "Node: "))
+         (node (completing-read
+                prompt
+                (lambda (string pred action)
+                  (if (eq action 'metadata)
+                      `(metadata
+                        ;; Preserve sorting in the completion UI if a sort-fn is used
+                        ,@(when sort-fn
+                            '((display-sort-function . identity)
+                              (cycle-sort-function . identity)))
+                        (annotation-function
+                         . ,(lambda (title)
+                              (funcall org-roam-node-annotation-function
+                                       (get-text-property 0 'node title))))
+                        (category . org-roam-node))
+                    (complete-with-action action nodes string pred)))
+                nil require-match initial-input 'org-roam-node-history)))
+    (or (cdr (assoc node nodes))
+        (org-roam-node-create :title node))))
+
+(defun org-roam-node-read--completions (&optional filter-fn sort-fn)
+  "Return an alist for node completion.
+The car is the displayed title or alias for the node, and the cdr
+is the `org-roam-node'.
+FILTER-FN is a function to filter out nodes: it takes an `org-roam-node',
+and when nil is returned the node will be filtered out.
+SORT-FN is a function to sort nodes. See `org-roam-node-read-sort-by-file-mtime'
+for an example sort function.
+The displayed title is formatted according to `org-roam-node-display-template'."
+  (let* ((template (org-roam-node--process-display-format org-roam-node-display-template))
+         (nodes (org-roam-node-list))
+         (nodes (mapcar (lambda (node)
+                          (org-roam-node-read--to-candidate node template)) nodes))
+         (nodes (if filter-fn
+                    (cl-remove-if-not
+                     (lambda (n) (funcall filter-fn (cdr n)))
+                     nodes)
+                  nodes))
          (sort-fn (or sort-fn
                       (when org-roam-node-default-sort
                         (intern (concat "org-roam-node-read-sort-by-"
                                         (symbol-name org-roam-node-default-sort))))))
-         (_ (when sort-fn (setq nodes (seq-sort sort-fn nodes))))
-         (node (completing-read
-                "Node: "
-                (lambda (string pred action)
-                  (if (eq action 'metadata)
-                      '(metadata
-                        (annotation-function . (lambda (title)
-                                                 (funcall org-roam-node-annotation-function
-                                                          (get-text-property 0 'node title))))
-                        (category . org-roam-node))
-                    (complete-with-action action nodes string pred)))
-                nil require-match initial-input)))
-    (or (cdr (assoc node nodes))
-        (org-roam-node-create :title node))))
+         (nodes (if sort-fn (seq-sort sort-fn nodes)
+                  nodes)))
+    nodes))
 
-(defvar org-roam-node-read--cached-display-format nil)
-
-(defun org-roam-node-read--completions ()
-  "Return an alist for node completion.
-The car is the displayed title or alias for the node, and the cdr
-is the `org-roam-node'.
-The displayed title is formatted according to `org-roam-node-display-template'."
-  (setq org-roam-node-read--cached-display-format nil)
-  (let ((nodes (org-roam-node-list)))
-    (mapcar #'org-roam-node-read--to-candidate nodes)))
-
-(defun org-roam-node-read--to-candidate (node)
-  "Return a minibuffer completion candidate given NODE."
-  (let ((candidate-main (org-roam-node-read--format-entry node (1- (frame-width)))))
+(defun org-roam-node-read--to-candidate (node template)
+  "Return a minibuffer completion candidate given NODE.
+TEMPLATE is the processed template used to format the entry."
+  (let ((candidate-main (org-roam-node--format-entry
+                         template
+                         node
+                         (1- (if (bufferp (current-buffer))
+                                 (window-width) (frame-width))))))
     (cons (propertize candidate-main 'node node) node)))
 
-(defun org-roam-node-read--format-entry (node width)
+(defun org-roam-node--format-entry (template node &optional width)
   "Formats NODE for display in the results list.
 WIDTH is the width of the results list.
-Uses `org-roam-node-display-template' to format the entry."
-  (pcase-let ((`(,tmpl . ,tmpl-width)
-               (org-roam-node-read--process-display-format org-roam-node-display-template)))
+TEMPLATE is the processed template used to format the entry."
+  (pcase-let ((`(,tmpl . ,tmpl-width) template))
     (org-roam-format-template
      tmpl
      (lambda (field _default-val)
@@ -508,42 +602,45 @@ Uses `org-roam-node-display-template' to format the entry."
                             ((not field-width)
                              field-width)
                             ((string-equal field-width "*")
-                             (- width tmpl-width))
+                             (if width
+                                 (- width tmpl-width)
+                               tmpl-width))
                             ((>= (string-to-number field-width) 0)
                              (string-to-number field-width))))
-         ;; Setting the display (which would be padded out to the field length) for an
-         ;; empty string results in an empty string and misalignment for candidates that
-         ;; don't have some field. This uses the actual display string, made of spaces
-         ;; when the field-value is "" so that we actually take up space.
-         (let ((display-string (if field-width
-                                   (truncate-string-to-width field-value field-width 0 ?\s)
-                                 field-value)))
-           (if (equal field-value "")
-               display-string
-             ;; Remove properties from the full candidate string, otherwise the display
-             ;; formatting with pre-prioritized field-values gets messed up.
-             (propertize (substring-no-properties field-value) 'display display-string))))))))
+         (when field-width
+           (let* ((truncated (truncate-string-to-width field-value field-width 0 ?\s))
+                  (tlen (length truncated))
+                  (len (length field-value)))
+             (if (< tlen len)
+                 ;; Make the truncated part of the string invisible. If strings
+                 ;; are pre-propertized with display or invisible properties, the
+                 ;; formatting may get messed up. Ideally, truncated strings are
+                 ;; not preformatted with these properties. Face properties are
+                 ;; allowed without restriction.
+                 (put-text-property tlen len 'invisible t field-value)
+               ;; If the string wasn't truncated, but padded, use this string instead.
+               (setq field-value truncated))))
+         field-value)))))
 
-(defun org-roam-node-read--process-display-format (format)
+(defun org-roam-node--process-display-format (format)
   "Pre-calculate minimal widths needed by the FORMAT string."
-  (or org-roam-node-read--cached-display-format
-      (setq org-roam-node-read--cached-display-format
-            (let* ((fields-width 0)
-                   (string-width
-                    (string-width
-                     (org-roam-format-template
-                      format
-                      (lambda (field _default-val)
-                        (setq fields-width
-                              (+ fields-width
-                                 (string-to-number
-                                  (or (cadr (split-string field ":"))
-                                      "")))))))))
-              (cons format (+ fields-width string-width))))))
+  (let* ((fields-width 0)
+         (string-width
+          (string-width
+           (org-roam-format-template
+            format
+            (lambda (field _default-val)
+              (setq fields-width
+                    (+ fields-width
+                       (string-to-number
+                        (or (cadr (split-string field ":"))
+                            "")))))))))
+    (cons format (+ fields-width string-width))))
 
 (defun org-roam-node-read-sort-by-file-mtime (completion-a completion-b)
   "Sort files such that files modified more recently are shown first.
-COMPLETION-A and COMPLETION-B are items in the form of (node-title org-roam-node-struct)"
+COMPLETION-A and COMPLETION-B are items in the form of
+\(node-title org-roam-node-struct)"
   (let ((node-a (cdr completion-a))
         (node-b (cdr completion-b)))
     (time-less-p (org-roam-node-file-mtime node-b)
@@ -551,7 +648,8 @@ COMPLETION-A and COMPLETION-B are items in the form of (node-title org-roam-node
 
 (defun org-roam-node-read-sort-by-file-atime (completion-a completion-b)
   "Sort files such that files accessed more recently are shown first.
-COMPLETION-A and COMPLETION-B are items in the form of (node-title org-roam-node-struct)"
+COMPLETION-A and COMPLETION-B are items in the form of
+\(node-title org-roam-node-struct)"
   (let ((node-a (cdr completion-a))
         (node-b (cdr completion-b)))
     (time-less-p (org-roam-node-file-atime node-b)
@@ -583,16 +681,20 @@ The INFO, if provided, is passed to the underlying `org-roam-capture-'."
                     (setq region-text (org-link-display-format (buffer-substring-no-properties beg end)))))
                (node (org-roam-node-read region-text filter-fn))
                (description (or region-text
-                                (org-roam-node-title node))))
+                                (org-roam-node-formatted node))))
           (if (org-roam-node-id node)
               (progn
                 (when region-text
                   (delete-region beg end)
                   (set-marker beg nil)
                   (set-marker end nil))
-                (insert (org-link-make-string
-                         (concat "id:" (org-roam-node-id node))
-                         description)))
+                (let ((id (org-roam-node-id node)))
+                  (insert (org-link-make-string
+                           (concat "id:" id)
+                           description))
+                  (run-hook-with-args 'org-roam-post-node-insert-hook
+                                      id
+                                      description)))
             (org-roam-capture-
              :node node
              :info info
@@ -600,31 +702,10 @@ The INFO, if provided, is passed to the underlying `org-roam-capture-'."
              :props (append
                      (when (and beg end)
                        (list :region (cons beg end)))
-                     (list :insert-at (point-marker)
-                           :link-description description
+                     (list :link-description description
                            :finalize 'insert-link))))))
     (deactivate-mark)))
 
-(add-hook 'org-roam-find-file-hook #'org-roam-open-id-with-org-roam-db-h)
-(defun org-roam-open-id-with-org-roam-db-h ()
-  "Try to open \"id:\" links at point by querying them to the database."
-  (add-hook 'org-open-at-point-functions #'org-roam-open-id-at-point nil t))
-
-(defun org-roam-open-id-at-point ()
-  "Try to navigate \"id:\" link to find and visit node with an assigned ID.
-Assumes that the cursor was put where the link is."
-  (let* ((context (org-element-context))
-         (type (org-element-property :type context))
-         (id (org-element-property :path context)))
-    (when (string= type "id")
-      (let ((node (org-roam-populate (org-roam-node-create :id id))))
-        (cond
-         ((org-roam-node-file node)
-          (org-mark-ring-push)
-          (org-roam-node-visit node nil 'force)
-          t)
-         (t nil))))))
-
 ;;;;; [roam:] link
 (org-link-set-parameters "roam" :follow #'org-roam-link-follow-link)
 (defun org-roam-link-follow-link (title-or-alias)
@@ -676,7 +757,7 @@ Assumes that the cursor was put where the link is."
 
 ;;;;;; Completion-at-point interface
 (defconst org-roam-bracket-completion-re
-  "\\[\\[\\(\\(?:roam:\\)?\\)\\([^z-a]*\\)]]"
+  "\\[\\[\\(\\(?:roam:\\)?\\)\\([^z-a]*?\\)]]"
   "Regex for completion within link brackets.
 We use this as a substitute for `org-link-bracket-re', because
 `org-link-bracket-re' requires content within the brackets for a match.")
@@ -689,9 +770,7 @@ We use this as a substitute for `org-link-bracket-re', because
             start (match-beginning 2)
             end (match-end 2))
       (list start end
-            (completion-table-dynamic
-             (lambda (_)
-               (funcall #'org-roam--get-titles)))
+            (org-roam--get-titles)
             :exit-function
             (lambda (str &rest _)
               (delete-char (- 0 (length str)))
@@ -712,22 +791,23 @@ hence \"everywhere\"."
              (not (save-match-data (org-in-regexp org-link-any-re))))
     (let ((bounds (bounds-of-thing-at-point 'word)))
       (list (car bounds) (cdr bounds)
-            (completion-table-dynamic
-             (lambda (_)
-               (funcall #'org-roam--get-titles)))
+            (org-roam--get-titles)
             :exit-function
             (lambda (str _status)
               (delete-char (- (length str)))
-              (insert "[[roam:" str "]]"))))))
-
-(defun org-roam-complete-at-point ()
-  "Try get completion candidates at point using `org-roam-completion-functions'."
-  (run-hook-with-args-until-success 'org-roam-completion-functions))
+              (insert "[[roam:" str "]]"))
+            ;; Proceed with the next completion function if the returned titles
+            ;; do not match. This allows the default Org capfs or custom capfs
+            ;; of lower priority to run.
+            :exclusive 'no))))
 
 (add-hook 'org-roam-find-file-hook #'org-roam--register-completion-functions-h)
+(add-hook 'org-roam-indirect-buffer-hook #'org-roam--register-completion-functions-h)
+
 (defun org-roam--register-completion-functions-h ()
   "Setup `org-roam-completion-functions' for `completion-at-point'."
-  (add-hook 'completion-at-point-functions #'org-roam-complete-at-point nil t))
+  (dolist (f org-roam-completion-functions)
+    (add-hook 'completion-at-point-functions f nil t)))
 
 ;;;; Editing
 (defun org-roam-demote-entire-buffer ()
@@ -741,7 +821,8 @@ Any tags declared on #+FILETAGS: are transferred to tags on the new top heading.
 Any top level properties drawers are incorporated into the new heading."
   (interactive)
   (org-with-point-at 1
-    (org-map-entries 'org-do-demote)
+    (org-map-region #'org-do-demote
+                    (point-min) (point-max))
     (insert "* "
             (org-roam--get-keyword "title")
             "\n")
@@ -750,21 +831,40 @@ Any top level properties drawers are incorporated into the new heading."
     (org-roam-erase-keyword "title")
     (org-roam-erase-keyword "filetags")))
 
+(defun org-roam--h1-count ()
+  "Count level-1 headings in the current file."
+  (let ((h1-count 0))
+    (org-with-wide-buffer
+     (org-map-region (lambda ()
+                       (if (= (org-current-level) 1)
+                           (incf h1-count)))
+                     (point-min) (point-max))
+     h1-count)))
+
+(defun org-roam--buffer-promoteable-p ()
+  "Verify that this buffer is promoteable:
+There is a single level-1 heading
+and no extra content before the first heading."
+  (and
+   (= (org-roam--h1-count) 1)
+   (org-with-point-at 1 (org-at-heading-p))))
+
 (defun org-roam-promote-entire-buffer ()
   "Promote the current buffer.
-Converts a file containing a headline node at the top to a file
+Converts a file containing a single level-1 headline node to a file
 node."
   (interactive)
+  (unless (org-roam--buffer-promoteable-p)
+    (user-error "Cannot promote: multiple root headings or there is extra file-level text"))
   (org-with-point-at 1
-    (org-map-entries (lambda ()
-                       (when (> (org-outline-level) 1)
-                         (org-do-promote))))
     (let ((title (nth 4 (org-heading-components)))
-          (tags (nth 5 (org-heading-components))))
-      (beginning-of-line)
-      (kill-line 1)
-      (org-roam-set-keyword "title" title)
-      (when tags (org-roam-set-keyword "filetags" tags)))))
+          (tags (org-get-tags)))
+      (kill-whole-line)
+      (org-roam-end-of-meta-data)
+      (insert "#+title: " title "\n")
+      (when tags (org-roam-tag-add tags))
+      (org-map-region #'org-promote (point-min) (point-max))
+      (org-roam-db-update-file))))
 
 ;;;###autoload
 (defun org-roam-refile ()
@@ -779,48 +879,50 @@ If region is active, then use it instead of the node at point."
          (nbuf (or (find-buffer-visiting file)
                    (find-file-noselect file)))
          level reversed)
-    (if regionp
+    (if (equal (org-roam-node-at-point) node)
+        (user-error "Target is the same as current node")
+      (if regionp
+          (progn
+            (org-kill-new (buffer-substring region-start region-end))
+            (org-save-markers-in-region region-start region-end))
         (progn
-          (org-kill-new (buffer-substring region-start region-end))
-          (org-save-markers-in-region region-start region-end))
-      (progn
-        (if (org-before-first-heading-p)
-            (org-roam-demote-entire-buffer))
-        (org-copy-subtree 1 nil t)))
-    (with-current-buffer nbuf
-      (org-with-wide-buffer
-       (goto-char (org-roam-node-point node))
-       (setq level (org-get-valid-level (funcall outline-level) 1)
-             reversed (org-notes-order-reversed-p))
-       (goto-char
-        (if reversed
-            (or (outline-next-heading) (point-max))
-          (or (save-excursion (org-get-next-sibling))
-              (org-end-of-subtree t t)
-              (point-max))))
-       (unless (bolp) (newline))
-       (org-paste-subtree level nil nil t)
-       (and org-auto-align-tags
-            (let ((org-loop-over-headlines-in-active-region nil))
-              (org-align-tags)))
-       (when (fboundp 'deactivate-mark) (deactivate-mark))))
-    (if regionp
-        (delete-region (point) (+ (point) (- region-end region-start)))
-      (org-preserve-local-variables
-       (delete-region
-        (and (org-back-to-heading t) (point))
-        (min (1+ (buffer-size)) (org-end-of-subtree t t) (point)))))
-    ;; If the buffer end-up empty after the refile, kill it and delete its
-    ;; associated file.
-    (when (eq (buffer-size) 0)
-      (if (buffer-file-name)
-          (delete-file (buffer-file-name)))
-      (set-buffer-modified-p nil)
-      ;; In this was done during capture, abort the capture process.
-      (when (and org-capture-mode
-                 (buffer-base-buffer (current-buffer)))
-        (org-capture-kill))
-      (kill-buffer (current-buffer)))))
+          (if (org-before-first-heading-p)
+              (org-roam-demote-entire-buffer))
+          (org-copy-subtree 1 nil t)))
+      (with-current-buffer nbuf
+        (org-with-wide-buffer
+         (goto-char (org-roam-node-point node))
+         (setq level (org-get-valid-level (funcall outline-level) 1)
+               reversed (org-notes-order-reversed-p))
+         (goto-char
+          (if reversed
+              (or (outline-next-heading) (point-max))
+            (or (save-excursion (org-get-next-sibling))
+                (org-end-of-subtree t t)
+                (point-max))))
+         (unless (bolp) (newline))
+         (org-paste-subtree level nil nil t)
+         (and org-auto-align-tags
+              (let ((org-loop-over-headlines-in-active-region nil))
+                (org-align-tags)))
+         (when (fboundp 'deactivate-mark) (deactivate-mark))))
+      (if regionp
+          (delete-region (point) (+ (point) (- region-end region-start)))
+        (org-preserve-local-variables
+         (delete-region
+          (and (org-back-to-heading t) (point))
+          (min (1+ (buffer-size)) (org-end-of-subtree t t) (point)))))
+      ;; If the buffer end-up empty after the refile, kill it and delete its
+      ;; associated file.
+      (when (eq (buffer-size) 0)
+        (if (buffer-file-name)
+            (delete-file (buffer-file-name)))
+        (set-buffer-modified-p nil)
+        ;; In this was done during capture, abort the capture process.
+        (when (and org-capture-mode
+                   (buffer-base-buffer (current-buffer)))
+          (org-capture-kill))
+        (kill-buffer (current-buffer))))))
 
 ;;;###autoload
 (defun org-roam-extract-subtree ()
@@ -845,55 +947,24 @@ If region is active, then use it instead of the node at point."
                             (funcall fn node))
                            ((fboundp node-fn)
                             (funcall node-fn node))
-                           (t (let ((r (completing-read (format "%s: " key) nil nil nil default-val)))
+                           (t (let ((r (read-from-minibuffer (format "%s: " key) default-val)))
                                 (plist-put template-info ksym r)
                                 r)))))))
-           (file-path (read-file-name "Extract node to: " org-roam-directory template nil template)))
+           (file-path
+            (expand-file-name
+             (read-file-name "Extract node to: "
+                             (file-name-as-directory org-roam-directory) template nil template)
+             org-roam-directory)))
       (when (file-exists-p file-path)
         (user-error "%s exists. Aborting" file-path))
       (org-cut-subtree)
       (save-buffer)
       (with-current-buffer (find-file-noselect file-path)
         (org-paste-subtree)
+        (while (> (org-current-level) 1) (org-promote-subtree))
         (org-roam-promote-entire-buffer)
         (save-buffer)))))
 
-;;; IDs
-;;;; Getters
-(defun org-roam-id-at-point ()
-  "Return the ID at point, if any.
-Recursively traverses up the headline tree to find the
-first encapsulating ID."
-  (org-with-wide-buffer
-   (org-back-to-heading-or-point-min t)
-   (while (and (not (org-roam-db-node-p))
-               (not (bobp)))
-     (org-roam-up-heading-or-point-min))
-   (when (org-roam-db-node-p)
-     (org-id-get))))
-
-;;;###autoload
-(defun org-roam-update-org-id-locations (&rest directories)
-  "Scan Org-roam files to update `org-id' related state.
-This is like `org-id-update-id-locations', but will automatically
-use the currently bound `org-directory' and `org-roam-directory'
-along with DIRECTORIES (if any), where the lookup for files in
-these directories will be always recursive.
-
-Note: Org-roam doesn't have hard dependency on
-`org-id-locations-file' to lookup IDs for nodes that are stored
-in the database, but it still tries to properly integrates with
-`org-id'. This allows the user to cross-reference IDs outside of
-the current `org-roam-directory', and also link with \"id:\"
-links to headings/files within the current `org-roam-directory'
-that are excluded from identification in Org-roam as
-`org-roam-node's, e.g. with \"ROAM_EXCLUDE\" property."
-  (interactive)
-  (cl-loop with files for dir in (cons org-roam-directory directories)
-           for org-roam-directory = dir
-           nconc (org-roam-list-files) into files
-           finally (org-id-update-id-locations files org-roam-verbose)))
-
 ;;; Refs
 ;;;; Completing-read interface
 (defun org-roam-ref-read (&optional initial-input filter-fn)
@@ -908,13 +979,12 @@ filtered out."
          (ref (completing-read "Ref: "
                                (lambda (string pred action)
                                  (if (eq action 'metadata)
-                                     '(metadata
-                                       (annotation-function . (lambda (ref)
-                                                                (funcall org-roam-ref-annotation-function
-                                                                         ref)))
+                                     `(metadata
+                                       (annotation-function
+                                        . ,org-roam-ref-annotation-function)
                                        (category . org-roam-ref))
                                    (complete-with-action action refs string pred)))
-                               nil t initial-input)))
+                               nil t initial-input 'org-roam-ref-history)))
     (cdr (assoc ref refs))))
 
 (defun org-roam-ref-read--completions ()
@@ -959,7 +1029,7 @@ and when nil is returned the node will be filtered out."
   (let ((node (org-roam-node-at-point 'assert)))
     (save-excursion
       (goto-char (org-roam-node-point node))
-      (org-roam-add-property ref "ROAM_REFS"))))
+      (org-roam-property-add "ROAM_REFS" ref))))
 
 (defun org-roam-ref-remove (&optional ref)
   "Remove a REF from the node at point."
@@ -967,7 +1037,7 @@ and when nil is returned the node will be filtered out."
   (let ((node (org-roam-node-at-point 'assert)))
     (save-excursion
       (goto-char (org-roam-node-point node))
-      (org-roam-remove-property "ROAM_REFS" ref))))
+      (org-roam-property-remove "ROAM_REFS" ref))))
 
 ;;; Tags
 ;;;; Getters
@@ -1016,7 +1086,7 @@ and when nil is returned the node will be filtered out."
                                   (org-make-tag-string (seq-difference current-tags tags #'string-equal))))
         (let* ((current-tags (or (org-get-tags)
                                  (user-error "No tag to remove")))
-               (tags (completing-read-multiple "Tag: " current-tags)))
+               (tags (or tags (completing-read-multiple "Tag: " current-tags))))
           (org-set-tags (seq-difference current-tags tags #'string-equal))))
       tags)))
 
@@ -1034,7 +1104,7 @@ and when nil is returned the node will be filtered out."
   (let ((node (org-roam-node-at-point 'assert)))
     (save-excursion
       (goto-char (org-roam-node-point node))
-      (org-roam-add-property alias "ROAM_ALIASES"))))
+      (org-roam-property-add "ROAM_ALIASES" alias))))
 
 (defun org-roam-alias-remove (&optional alias)
   "Remove an ALIAS from the node at point."
@@ -1042,7 +1112,7 @@ and when nil is returned the node will be filtered out."
   (let ((node (org-roam-node-at-point 'assert)))
     (save-excursion
       (goto-char (org-roam-node-point node))
-      (org-roam-remove-property "ROAM_ALIASES" alias))))
+      (org-roam-property-remove "ROAM_ALIASES" alias))))
 
 
 (provide 'org-roam-node)

org-roam-utils.el

@@ -1,11 +1,11 @@
 ;;; org-roam-utils.el --- Utilities for Org-roam -*- lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
+;; Version: 2.2.1
 ;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4"))
 
 ;; This file is NOT part of GNU Emacs.
@@ -34,6 +34,11 @@
 
 (require 'org-roam)
 
+(defun org-roam-require (libs)
+  "Require LIBS."
+  (dolist (lib libs)
+    (require lib nil 'noerror)))
+
 ;;; String utilities
 ;; TODO Refactor this.
 (defun org-roam-replace-string (old new s)
@@ -44,23 +49,81 @@
 (defun org-roam-quote-string (s)
   "Quotes string S."
   (->> s
-    (org-roam-replace-string "\\" "\\\\")
-    (org-roam-replace-string "\"" "\\\"")))
+       (org-roam-replace-string "\\" "\\\\")
+       (org-roam-replace-string "\"" "\\\"")))
+
+(defun org-roam-word-wrap (len s)
+  "If S is longer than LEN, wrap the words with newlines."
+  (declare (side-effect-free t))
+  (save-match-data
+    (with-temp-buffer
+      (insert s)
+      (let ((fill-column len))
+        (fill-region (point-min) (point-max)))
+      (buffer-substring (point-min) (point-max)))))
+
+(defun org-roam-string-equal (s1 s2)
+  "Return t if S1 and S2 are equal.
+Like `string-equal', but case-insensitive."
+  (and (= (length s1) (length s2))
+       (or (string-equal s1 s2)
+           (string-equal (downcase s1) (downcase s2)))))
+
+(defun org-roam-whitespace-content (s)
+  "Return the whitespace content at the end of S."
+  (with-temp-buffer
+    (let ((c 0))
+      (insert s)
+      (skip-chars-backward " \t\n")
+      (buffer-substring-no-properties
+       (point) (point-max)))))
+
+(defun org-roam-strip-comments (s)
+  "Strip Org comments from string S."
+  (with-temp-buffer
+    (insert s)
+    (goto-char (point-min))
+    (while (not (eobp))
+      (if (org-at-comment-p)
+          (delete-region (point-at-bol) (progn (forward-line) (point)))
+        (forward-line)))
+    (buffer-string)))
 
 ;;; List utilities
-(defmacro org-roam-plist-map! (fn plist)
-  "Map FN over PLIST, modifying it in-place."
-  (declare (indent 1))
-  (let ((plist-var (make-symbol "plist"))
-        (k (make-symbol "k"))
-        (v (make-symbol "v")))
-    `(let ((,plist-var (copy-sequence ,plist)))
-       (while ,plist-var
-         (setq ,k (pop ,plist-var))
-         (setq ,v (pop ,plist-var))
-         (setq ,plist (plist-put ,plist ,k (funcall ,fn ,k ,v)))))))
+(defun org-roam-plist-map! (fn plist)
+  "Map FN over PLIST, modifying it in-place and returning it.
+FN must take two arguments: the key and the value."
+  (let ((plist-index plist))
+    (while plist-index
+      (let ((key (pop plist-index)))
+        (setf (car plist-index) (funcall fn key (car plist-index))
+              plist-index (cdr plist-index)))))
+  plist)
+
+(defmacro org-roam-dolist-with-progress (spec msg &rest body)
+  "Loop over a list and report progress in the echo area.
+Like `dolist-with-progress-reporter', but falls back to `dolist'
+if the function does not yet exist.
+
+Evaluate BODY with VAR bound to each car from LIST, in turn.
+Then evaluate RESULT to get return value, default nil.
+
+MSG is a progress reporter object or a string.  In the latter
+case, use this string to create a progress reporter.
+
+SPEC is a list, as per `dolist'."
+  (declare (indent 2))
+  (if (fboundp 'dolist-with-progress-reporter)
+      `(dolist-with-progress-reporter ,spec ,msg ,@body)
+    `(dolist ,spec ,@body)))
 
 ;;; File utilities
+(defun org-roam-descendant-of-p (a b)
+  "Return t if A is descendant of B."
+  (unless (equal (file-truename a) (file-truename b))
+    (string-prefix-p (replace-regexp-in-string "^\\([A-Za-z]\\):" 'downcase (expand-file-name b) t t)
+                     (replace-regexp-in-string "^\\([A-Za-z]\\):" 'downcase (expand-file-name a) t t))))
+
 (defmacro org-roam-with-file (file keep-buf-p &rest body)
   "Execute BODY within FILE.
 If FILE is nil, execute BODY in the current buffer.
@@ -68,6 +131,7 @@ Kills the buffer if KEEP-BUF-P is nil, and FILE is not yet visited."
   (declare (indent 2) (debug t))
   `(let* (new-buf
           (auto-mode-alist nil)
+          (find-file-hook nil)
           (buf (or (and (not ,file)
                         (current-buffer)) ;If FILE is nil, use current buffer
                    (find-buffer-visiting ,file) ; If FILE is already visited, find buffer
@@ -76,11 +140,12 @@ Kills the buffer if KEEP-BUF-P is nil, and FILE is not yet visited."
                      (find-file-noselect ,file)))) ; Else, visit FILE and return buffer
           res)
      (with-current-buffer buf
-       (unless (equal major-mode 'org-mode)
+       (unless (derived-mode-p 'org-mode)
          (delay-mode-hooks
            (let ((org-inhibit-startup t)
                  (org-agenda-files nil))
-             (org-mode))))
+             (org-mode)
+             (hack-local-variables))))
        (setq res (progn ,@body))
        (unless (and new-buf (not ,keep-buf-p))
          (save-buffer)))
@@ -127,14 +192,20 @@ value (possibly nil). Adapted from `s-format'."
                  (let ((v (progn
                             (set-match-data saved-match-data)
                             (funcall replacer var default-val))))
-                   (if v (format "%s" v) (signal 'org-roam-format-resolve md)))
+                   (if v
+                       (format (apply #'propertize "%s" (text-properties-at 0 var)) v)
+                     (signal 'org-roam-format-resolve md)))
                (set-match-data replacer-match-data))))
-         template
+         (if (functionp template)
+             (funcall template)
+           template)
          ;; Need literal to make sure it works
          t t)
       (set-match-data saved-match-data))))
 
 ;;; Fontification
+(defvar org-ref-buffer-hacked)
+
 (defun org-roam-fontify-like-in-org-mode (s)
   "Fontify string S like in Org mode.
 Like `org-fontify-like-in-org-mode', but supports `org-ref'."
@@ -223,6 +294,45 @@ If BOUND, scan up to BOUND bytes of the buffer."
       (when (re-search-forward re bound t)
         (buffer-substring-no-properties (match-beginning 1) (match-end 1))))))
 
+(defun org-roam-end-of-meta-data (&optional full)
+  "Like `org-end-of-meta-data', but supports file-level metadata.
+
+When FULL is non-nil but not t, skip planning information,
+properties, clocking lines and logbook drawers.
+
+When optional argument FULL is t, skip everything above, and also
+skip keywords."
+  (org-back-to-heading-or-point-min t)
+  (when (org-at-heading-p) (forward-line))
+  ;; Skip planning information.
+  (when (looking-at-p org-planning-line-re) (forward-line))
+  ;; Skip property drawer.
+  (when (looking-at org-property-drawer-re)
+    (goto-char (match-end 0))
+    (forward-line))
+  ;; When FULL is not nil, skip more.
+  (when (and full (not (org-at-heading-p)))
+    (catch 'exit
+      (let ((end (save-excursion (outline-next-heading) (point)))
+            (re (concat "[ \t]*$" "\\|" org-clock-line-re)))
+        (while (not (eobp))
+          (cond ;; Skip clock lines.
+           ((looking-at-p re) (forward-line))
+           ;; Skip logbook drawer.
+           ((looking-at-p org-logbook-drawer-re)
+            (if (re-search-forward "^[ \t]*:END:[ \t]*$" end t)
+                (forward-line)
+              (throw 'exit t)))
+           ((looking-at-p org-drawer-regexp)
+            (if (re-search-forward "^[ \t]*:END:[ \t]*$" end t)
+                (forward-line)
+              (throw 'exit t)))
+           ;; When FULL is t, skip keywords too.
+           ((and (eq full t)
+                 (looking-at-p org-keyword-regexp))
+            (forward-line))
+           (t (throw 'exit t))))))))
+
 (defun org-roam-set-keyword (key value)
   "Set keyword KEY to VALUE.
 If the property is already set, it's value is replaced."
@@ -232,14 +342,13 @@ If the property is already set, it's value is replaced."
           (if (string-blank-p value)
               (kill-whole-line)
             (replace-match (concat " " value) 'fixedcase nil nil 1))
-        (while (and (not (eobp))
-                    (looking-at "^[#:]"))
-          (if (save-excursion (end-of-line) (eobp))
-              (progn
-                (end-of-line)
-                (insert "\n"))
-            (forward-line)
-            (beginning-of-line)))
+        (org-roam-end-of-meta-data 'drawers)
+        (if (save-excursion (end-of-line) (eobp))
+            (progn
+              (end-of-line)
+              (insert "\n"))
+          (forward-line)
+          (beginning-of-line))
         (insert "#+" key ": " value "\n")))))
 
 (defun org-roam-erase-keyword (keyword)
@@ -276,6 +385,40 @@ If VAL is not specified, user is prompted to select a value."
       (org-delete-property prop))
     prop-to-remove))
 
+(defun org-roam-property-add (prop val)
+  "Add VAL value to PROP property for the node at point.
+Both, VAL and PROP are strings."
+  (let* ((p (org-entry-get (point) prop))
+         (lst (when p (split-string-and-unquote p)))
+         (lst (if (memq val lst) lst (cons val lst)))
+         (lst (seq-uniq lst)))
+    (org-set-property prop (combine-and-quote-strings lst))
+    val))
+
+(defun org-roam-property-remove (prop &optional val)
+  "Remove VAL value from PROP property for the node at point.
+Both VAL and PROP are strings.
+
+If VAL is not specified, user is prompted to select a value."
+  (let* ((p (org-entry-get (point) prop))
+         (lst (when p (split-string-and-unquote p)))
+         (prop-to-remove (or val (completing-read "Remove: " lst)))
+         (lst (delete prop-to-remove lst)))
+    (if lst
+        (org-set-property prop (combine-and-quote-strings lst))
+      (org-delete-property prop))
+    prop-to-remove))
+
+;;; Refs
+(defun org-roam-org-ref-path-to-keys (path)
+  "Return a list of keys given an org-ref cite: PATH.
+Accounts for both v2 and v3."
+  (cond ((fboundp 'org-ref-parse-cite-path)
+         (mapcar (lambda (cite) (plist-get cite :key))
+                 (plist-get (org-ref-parse-cite-path path) :references)))
+        ((fboundp 'org-ref-split-and-strip-string)
+         (org-ref-split-and-strip-string path))))
+
 ;;; Logs
 (defvar org-roam-verbose)
 (defun org-roam-message (format-string &rest args)
@@ -343,7 +486,8 @@ See <https://github.com/raxod502/straight.el/issues/520>."
                                          '("Doom" "Spacemacs" "N/A" "I don't know"))
                       (quit "N/A"))))
     (insert (format "- Org: %s\n" (org-version nil 'full)))
-    (insert (format "- Org-roam: %s" (org-roam-version)))))
+    (insert (format "- Org-roam: %s" (org-roam-version)))
+    (insert (format "- sqlite-connector: %s" org-roam-database-connector))))
 
 
 (provide 'org-roam-utils)

org-roam.el

@@ -1,12 +1,12 @@
 ;;; org-roam.el --- A database abstraction layer for Org-mode -*- coding: utf-8; lexical-binding: t; -*-
 
-;; Copyright © 2020-2021 Jethro Kuan <jethrokuan95@gmail.com>
+;; Copyright © 2020-2022 Jethro Kuan <jethrokuan95@gmail.com>
 
 ;; Author: Jethro Kuan <jethrokuan95@gmail.com>
 ;; URL: https://github.com/org-roam/org-roam
 ;; Keywords: org-mode, roam, convenience
-;; Version: 2.1.0
-;; Package-Requires: ((emacs "26.1") (dash "2.13") (f "0.17.2") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "2.90.1"))
+;; Version: 2.2.1
+;; Package-Requires: ((emacs "26.1") (dash "2.13") (org "9.4") (emacsql "3.0.0") (emacsql-sqlite "1.0.0") (magit-section "3.0.0"))
 
 ;; This file is NOT part of GNU Emacs.
 
@@ -71,7 +71,6 @@
 ;; majority of them can be found at https://github.com/org-roam and MELPA.
 ;;
 ;;; Code:
-(require 'f)
 (require 'dash)
 
 (require 'rx)
@@ -84,6 +83,7 @@
 (require 'emacsql-sqlite)
 
 (require 'org)
+(require 'org-attach)                   ; To set `org-attach-id-dir'
 (require 'org-id)
 (require 'ol)
 (require 'org-element)
@@ -94,8 +94,6 @@
 (eval-when-compile
   (require 'subr-x))
 
-(require 'org-roam-compat)
-
 ;;; Options
 (defgroup org-roam nil
   "A database abstraction layer for Org-mode."
@@ -125,6 +123,12 @@ All Org files, at any level of nesting, are considered part of the Org-roam."
   :group 'org-roam
   :type 'hook)
 
+(defcustom org-roam-post-node-insert-hook nil
+  "Hook run when an Org-roam node is inserted as an Org link.
+Each function takes two arguments: the id of the node, and the link description."
+  :group 'org-roam
+  :type 'hook)
+
 (defcustom org-roam-file-extensions '("org")
   "List of file extensions to be included by Org-Roam.
 While a file extension different from \".org\" may be used, the
@@ -133,9 +137,11 @@ responsibility to ensure that."
   :type '(repeat string)
   :group 'org-roam)
 
-(defcustom org-roam-file-exclude-regexp nil
+(defcustom org-roam-file-exclude-regexp (list org-attach-id-dir)
   "Files matching this regular expression are excluded from the Org-roam."
   :type '(choice
+          (repeat
+           (string :tag "Regular expression matching files to ignore"))
           (string :tag "Regular expression matching files to ignore")
           (const :tag "Include everything" nil))
   :group 'org-roam)
@@ -190,14 +196,25 @@ FILE is an Org-roam file if:
          (ext (when path (org-roam--file-name-extension path)))
          (ext (if (string= ext "gpg")
                   (org-roam--file-name-extension (file-name-sans-extension path))
-                ext)))
+                ext))
+         (org-roam-dir-p (org-roam-descendant-of-p path org-roam-directory))
+         (valid-file-ext-p (member ext org-roam-file-extensions))
+         (match-exclude-regexp-p
+          (cond
+           ((not org-roam-file-exclude-regexp) nil)
+           ((stringp org-roam-file-exclude-regexp)
+            (string-match-p org-roam-file-exclude-regexp path))
+           ((listp org-roam-file-exclude-regexp)
+            (let (is-match)
+              (dolist (exclude-re org-roam-file-exclude-regexp)
+                (setq is-match (or is-match (string-match-p exclude-re path))))
+              is-match)))))
     (save-match-data
       (and
        path
-       (member ext org-roam-file-extensions)
-       (not (and org-roam-file-exclude-regexp
-                 (string-match-p org-roam-file-exclude-regexp path)))
-       (f-descendant-of-p path (expand-file-name org-roam-directory))))))
+       org-roam-dir-p
+       valid-file-ext-p
+       (not match-exclude-regexp-p)))))
 
 (defun org-roam-list-files ()
   "Return a list of all Org-roam files under `org-roam-directory'.
@@ -262,7 +279,8 @@ If no files are found, an empty list is returned."
        (shell-command-to-string it)
        (ansi-color-filter-apply it)
        (split-string it "\n")
-       (seq-filter #'s-present? it)))
+       (seq-filter (lambda (s)
+                     (not (or (null s) (string= "" s)))) it)))
 
 (defun org-roam--list-files-search-globs (exts)
   "Given EXTS, return a list of search globs.
@@ -274,15 +292,15 @@ E.g. (\".org\") => (\"*.org\" \"*.org.gpg\")"
 (defun org-roam--list-files-find (executable dir)
   "Return all Org-roam files under DIR, using \"find\", provided as EXECUTABLE."
   (let* ((globs (org-roam--list-files-search-globs org-roam-file-extensions))
-         (names (s-join " -o " (mapcar (lambda (glob) (concat "-name " glob)) globs)))
-         (command (s-join " " `(,executable "-L" ,dir "-type f \\(" ,names "\\)"))))
+         (names (string-join (mapcar (lambda (glob) (concat "-name " glob)) globs) " -o "))
+         (command (string-join `(,executable "-L" ,dir "-type f \\(" ,names "\\)") " ")))
     (org-roam--shell-command-files command)))
 
 (defun org-roam--list-files-fd (executable dir)
   "Return all Org-roam files under DIR, using \"fd\", provided as EXECUTABLE."
   (let* ((globs (org-roam--list-files-search-globs org-roam-file-extensions))
-         (extensions (s-join " -e " (mapcar (lambda (glob) (substring glob 2 -1)) globs)))
-         (command (s-join " " `(,executable "-L" ,dir "--type file" ,extensions))))
+         (extensions (string-join (mapcar (lambda (glob) (concat "-e " (substring glob 2 -1))) globs) " "))
+         (command (string-join `(,executable "-L" "--type file" ,extensions "." ,dir) " ")))
     (org-roam--shell-command-files command)))
 
 (defalias 'org-roam--list-files-fdfind #'org-roam--list-files-fd)
@@ -290,10 +308,12 @@ E.g. (\".org\") => (\"*.org\" \"*.org.gpg\")"
 (defun org-roam--list-files-rg (executable dir)
   "Return all Org-roam files under DIR, using \"rg\", provided as EXECUTABLE."
   (let* ((globs (org-roam--list-files-search-globs org-roam-file-extensions))
-         (command (s-join " " `(,executable "-L" ,dir "--files"
-                                            ,@(mapcar (lambda (glob) (concat "-g " glob)) globs)))))
+         (command (string-join `(,executable "-L" ,dir "--files"
+                                             ,@(mapcar (lambda (glob) (concat "-g " glob)) globs)) " ")))
     (org-roam--shell-command-files command)))
 
+(declare-function org-roam--directory-files-recursively "org-roam-compat")
+
 (defun org-roam--list-files-elisp (dir)
   "Return all Org-roam files under DIR, using Elisp based implementation."
   (let ((regex (concat "\\.\\(?:"(mapconcat
@@ -309,11 +329,14 @@ E.g. (\".org\") => (\"*.org\" \"*.org.gpg\")"
 (provide 'org-roam)
 
 (cl-eval-when (load eval)
+  (require 'org-roam-compat)
   (require 'org-roam-utils)
   (require 'org-roam-db)
   (require 'org-roam-node)
+  (require 'org-roam-id)
   (require 'org-roam-capture)
   (require 'org-roam-mode)
+  (require 'org-roam-log)
   (require 'org-roam-migrate))
 
 ;;; org-roam.el ends here

tests/test-org-roam-capture.el

@@ -0,0 +1,56 @@
+;;; test-org-roam-capture.el --- Tests for Org-roam -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2020 Jethro Kuan
+
+;; Author: Jethro Kuan <jethrokuan95@gmail.com>
+;; Package-Requires: ((buttercup))
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;;; Code:
+
+(require 'buttercup)
+(require 'org-roam)
+
+(describe "org-roam-capture--fill-template"
+  (it "fills template without newline"
+    (expect
+     (org-roam-capture--fill-template "foo")
+     :to-equal "foo"))
+
+  (it "fills template ensuring newline"
+    (expect
+     (org-roam-capture--fill-template "foo" 'ensure-newline)
+     :to-equal "foo\n"))
+
+  (it "fills template with newline"
+    (expect
+     (org-roam-capture--fill-template "foo\n")
+     :to-equal "foo\n"))
+
+  (it "fills template with two newlines"
+    (expect
+     (org-roam-capture--fill-template "foo\n\n")
+     :to-equal "foo\n\n")
+    (expect
+     (org-roam-capture--fill-template "foo\n\t\n")
+     :to-equal "foo\n\t\n"))
+
+  (it "expands templates when it's a function"
+    (expect
+     (org-roam-capture--fill-template (lambda () "foo"))
+     :to-equal "foo")))
+
+(provide 'test-org-roam-capture)

tests/test-org-roam-utils.el

@@ -0,0 +1,39 @@
+;;; test-org-roam-utils.el --- Tests for Org-roam -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2020 Jethro Kuan
+
+;; Author: Jethro Kuan <jethrokuan95@gmail.com>
+;; Package-Requires: ((buttercup))
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+;;; Code:
+
+(require 'buttercup)
+(require 'org-roam)
+
+(describe "org-roam-whitespace-content"
+  (it "extracts whitespace correctly"
+    (expect
+     (org-roam-whitespace-content "foo")
+     :to-equal "")
+    (expect
+     (org-roam-whitespace-content "foo\n")
+     :to-equal "\n")
+    (expect
+     (org-roam-whitespace-content "foo\n\t\n")
+     :to-equal "\n\t\n")))
+
+(provide 'test-org-roam-utils)