refactor!(tree-sitter): replace tree-sitter w/ treesit

BREAKING CHANGE: This rewrites the :tools tree-sitter module to use
treesit instead of tree-sitter.el. Users will need to adapt to treesit
and remove any tree-sitter-specific config in their dotfiles.

Ref: #7623
Ref: #7742
Ref: #8197

modules/tools/tree-sitter/README.org

@@ -27,8 +27,10 @@ It includes:
 /This module has no flags./
 
 ** Packages
+- if [[doom-module:+compat]]
   - [[doom-package:evil-textobj-tree-sitter]] if [[doom-module::editor evil +everywhere]]
   - [[doom-package:tree-sitter]]
+  - [[doom-package:tree-sitter-indent]]
   - [[doom-package:tree-sitter-langs]]
 
 ** Hacks

modules/tools/tree-sitter/autoload.el

@@ -1,41 +0,0 @@
-;;; tools/tree-sitter/autoload.el -*- lexical-binding: t; -*-
-
-;;;###autodef (fset 'tree-sitter! #'ignore)
-(defun tree-sitter! ()
-  "Dispatch to turn on tree sitter.
-
-Used as a hook function which turns on `tree-sitter-mode'
-and selectively turn on `tree-sitter-hl-mode'.
-according to `+tree-sitter-hl-enabled-modes'"
-  (turn-on-tree-sitter-mode)
-  ;; conditionally enable `tree-sitter-hl-mode'
-  (let ((mode (bound-and-true-p tree-sitter-hl-mode)))
-    (when-let (mode (if (pcase +tree-sitter-hl-enabled-modes
-                          (`(not . ,modes) (not (memq major-mode modes)))
-                          ((and `(,_ . ,_) modes) (memq major-mode modes))
-                          (bool bool))
-                        (unless mode +1)
-                      (if mode -1)))
-      (tree-sitter-hl-mode mode))))
-
-;;;###autodef (fset 'set-tree-sitter-lang! #'ignore)
-(defun set-tree-sitter-lang! (mode lang)
-  "Associate LANG with major MODE."
-  (after! tree-sitter-langs
-    (add-to-list 'tree-sitter-major-mode-language-alist (cons mode lang))))
-
-;; HACK: Remove and refactor when `use-package' eager macro expansion is solved or `use-package!' is removed
-;;;###autoload
-(defun +tree-sitter-get-textobj (group &optional query)
-  "A wrapper around `evil-textobj-tree-sitter-get-textobj' to
-prevent eager expansion."
-  (eval `(evil-textobj-tree-sitter-get-textobj ,group ,query)))
-
-;;;###autoload
-(defun +tree-sitter-goto-textobj (group &optional previous end query)
-  "Thin wrapper that returns the symbol of a named function, used in keybindings."
-  (let ((sym (intern (format "+goto%s%s-%s" (if previous "-previous" "") (if end "-end" "") group))))
-    (fset sym (lambda ()
-                (interactive)
-                (evil-textobj-tree-sitter-goto-textobj group previous end query)))
-    sym))

modules/tools/tree-sitter/autoload/compat-30.el

@@ -0,0 +1,43 @@
+;;; tools/tree-sitter/autoload/compat-30.el -*- lexical-binding: t; -*-
+;;;###if (and (fboundp 'treesit-available-p) (version< emacs-version "31.1"))
+;;
+;; Backported from 31.1
+;;
+;;; Code:
+
+(autoload 'treesit-ready-p "treesit")
+
+;;;###autoload
+(defcustom treesit-auto-install-grammar 'ask
+  "Whether to install tree-sitter language grammar libraries when needed.
+This controls whether Emacs will install missing grammar libraries
+when they are needed by some tree-sitter based mode.
+If `ask', ask for confirmation before installing the required grammar library.
+If `always', install the grammar library without asking.
+If nil or `never' or anything else, don't install the grammar library
+even while visiting a file in the mode that requires such grammar; this
+might display a warning and/or fail to turn on the mode."
+  :type '(choice (const :tag "Never install grammar libraries" never)
+          (const :tag "Always automatically install grammar libraries"
+                 always)
+          (const :tag "Ask whether to install missing grammar libraries"
+                 ask))
+  :version "31.1"
+  :group 'treesit)
+
+;;;###autoload
+(defun treesit-ensure-installed (lang)
+  "Ensure that the grammar library for the language LANG is installed.
+The option `treesit-auto-install-grammar' defines whether to install
+the grammar library if it's unavailable."
+  (or (treesit-ready-p lang t)
+      (when (or (eq treesit-auto-install-grammar 'always)
+                (and (eq treesit-auto-install-grammar 'ask)
+                     (y-or-n-p (format "\
+Tree-sitter grammar for `%s' is missing; install it?"
+                                       lang))))
+        (treesit-install-language-grammar lang)
+        ;; Check that the grammar was installed successfully
+        (treesit-ready-p lang))))
+
+;;; compat-30.el ends here

modules/tools/tree-sitter/autoload/tree-sitter.el

@@ -0,0 +1,71 @@
+;;; tools/tree-sitter/autoload/tree-sitter.el -*- lexical-binding: t; -*-
+
+;;;###autodef (fset 'tree-sitter! #'ignore)
+(defun tree-sitter! ()
+  (message "Old tree-sitter.el support is deprecated!"))
+
+;;;###autodef (fset 'set-tree-sitter! #'ignore)
+(defun set-tree-sitter! (mode ts-mode &optional langs)
+  "Remap major MODE to TS-MODE.
+
+MODE and TS-MODE are major mode symbols. If LANGS is provided, fall back to MODE
+if LANGS don't pass `treesit-ready-p' when activating TS-MODE. Use this for ts
+modes that error out instead of failing gracefully."
+  (declare (indent 2))
+  (cl-check-type mode symbol)
+  (cl-check-type ts-mode symbol)
+  (setq langs (ensure-list langs))
+  (dolist (m (ensure-list mode))
+    (add-to-list
+     'major-mode-remap-defaults
+     (cons
+      m (let (ensured?)
+          (lambda ()
+            (funcall
+             ;; Because standard major-mode remapping doesn't offer graceful
+             ;; failure in some cases, I implement it myself:
+             (cond ((null langs) m)
+                   ((not (fboundp ts-mode))
+                    (message "Couldn't find %S, using %S instead" ts-mode m)
+                    m)
+                   ((and (fboundp 'treesit-available-p)
+                         (treesit-available-p)
+                         (fboundp ts-mode)
+                         ;; Only prompt once, and log other times.
+                         (cl-every (if ensured?
+                                       (doom-rpartial #'treesit-ready-p 'message)
+                                     #'treesit-ensure-installed)
+                                   (cl-loop for lang in langs
+                                            if (listp lang)
+                                            collect (car lang)
+                                            else collect (list lang))))
+                    ts-mode)
+                   ((setq ensured? t)
+                    m))))))))
+  (with-eval-after-load 'treesit
+    (dolist (lang langs)
+      (when (and lang (listp lang))
+        (cl-destructuring-bind (name &key url rev source-dir cc cpp commit) lang
+          (setf (alist-get name treesit-language-source-alist)
+                (list url rev source-dir cc cpp commit)))))))
+
+;; ;; HACK: Remove and refactor when `use-package' eager macro expansion is solved or `use-package!' is removed
+;; ;;;###autoload
+;; (defun +tree-sitter-get-textobj (group &optional query)
+;;   "A wrapper around `evil-textobj-tree-sitter-get-textobj' to
+;; prevent eager expansion."
+;;   (eval `(evil-textobj-tree-sitter-get-textobj ,group ,query)))
+
+;; ;;;###autoload
+;; (defun +tree-sitter-goto-textobj (group &optional previous end query)
+;;   "Thin wrapper that returns the symbol of a named function, used in keybindings."
+;;   (let ((sym (intern (format "+goto%s%s-%s" (if previous "-previous" "") (if end "-end" "") group))))
+;;     (fset sym (lambda ()
+;;                 (interactive)
+;;                 (evil-textobj-tree-sitter-goto-textobj group previous end query)))
+;;     sym))
+
+
+;;; TODO: Backwards compatibility
+
+;;; tree-sitter.el ends here

modules/tools/tree-sitter/config.el

@@ -1,77 +1,135 @@
 ;;; tools/tree-sitter/config.el -*- lexical-binding: t; -*-
 
-(defvar +tree-sitter-hl-enabled-modes '(not web-mode typescript-tsx-mode)
-  "A list of major modes which should be highlighted by tree-sitter.
-
-If this list begins with `not', then it negates the list.
-If it is t, it is enabled in all modes.
-If nil, it is disabled in all modes")
-
 ;;
 ;;; Packages
 
-(use-package! tree-sitter
+(use-package! treesit
+  :when (fboundp 'treesit-available-p)
+  :when (treesit-available-p)
   :defer t
   :config
-  (require 'tree-sitter-langs)
+  ;; HACK: treesit lacks any way to dictate where to install grammars.
-  ;; This makes every node a link to a section of code
+  (defadvice! +tree-sitter--install-grammar-to-local-dir-a (fn &rest args)
-  (setq tree-sitter-debug-jump-buttons t
+    "Write grammars to `doom-profile-data-dir'."
-        ;; and this highlights the entire sub tree in your code
+    :around #'treesit-install-language-grammar
-        tree-sitter-debug-highlight-jump-region t))
+    :around #'treesit--build-grammar
+    (let ((user-emacs-directory doom-profile-data-dir))
+      (apply fn args)))
+
+  ;; TODO: Move most of these out to modules
+  (dolist (map '((awk "https://github.com/Beaglefoot/tree-sitter-awk" nil nil nil nil)
+                 (bibtex "https://github.com/latex-lsp/tree-sitter-bibtex" nil nil nil nil)
+                 (blueprint "https://github.com/huanie/tree-sitter-blueprint" nil nil nil nil)
+                 (c "https://github.com/tree-sitter/tree-sitter-c" nil nil nil nil)
+                 (c-sharp "https://github.com/tree-sitter/tree-sitter-c-sharp" nil nil nil nil)
+                 (clojure "https://github.com/sogaiu/tree-sitter-clojure" nil nil nil nil)
+                 (cmake "https://github.com/uyha/tree-sitter-cmake" nil nil nil nil)
+                 (commonlisp "https://github.com/tree-sitter-grammars/tree-sitter-commonlisp" nil nil nil nil)
+                 (cpp "https://github.com/tree-sitter/tree-sitter-cpp" nil nil nil nil)
+                 (css "https://github.com/tree-sitter/tree-sitter-css" nil nil nil nil)
+                 (dart "https://github.com/ast-grep/tree-sitter-dart" nil nil nil nil)
+                 (dockerfile "https://github.com/camdencheek/tree-sitter-dockerfile" nil nil nil nil)
+                 (elixir "https://github.com/elixir-lang/tree-sitter-elixir" nil nil nil nil)
+                 (glsl "https://github.com/tree-sitter-grammars/tree-sitter-glsl" nil nil nil nil)
+                 (go "https://github.com/tree-sitter/tree-sitter-go" nil nil nil nil)
+                 (gomod "https://github.com/camdencheek/tree-sitter-go-mod" nil nil nil nil)
+                 (heex "https://github.com/phoenixframework/tree-sitter-heex" nil nil nil nil)
+                 (html "https://github.com/tree-sitter/tree-sitter-html" nil nil nil nil)
+                 (java "https://github.com/tree-sitter/tree-sitter-java" nil nil nil nil)
+                 (javascript "https://github.com/tree-sitter/tree-sitter-javascript" "master" "src" nil nil)
+                 (json "https://github.com/tree-sitter/tree-sitter-json" nil nil nil nil)
+                 (julia "https://github.com/tree-sitter/tree-sitter-julia" nil nil nil nil)
+                 (kotlin "https://github.com/fwcd/tree-sitter-kotlin" nil nil nil nil)
+                 (latex "https://github.com/latex-lsp/tree-sitter-latex" nil nil nil nil)
+                 (lua "https://github.com/tree-sitter-grammars/tree-sitter-lua" nil nil nil nil)
+                 (make "https://github.com/tree-sitter-grammars/tree-sitter-make" nil nil nil nil)
+                 (nix "https://github.com/nix-community/tree-sitter-nix" nil nil nil nil)
+                 (nu "https://github.com/nushell/tree-sitter-nu" nil nil nil nil)
+                 (org "https://github.com/milisims/tree-sitter-org" nil nil nil nil)
+                 (perl "https://github.com/ganezdragon/tree-sitter-perl" nil nil nil nil)
+                 (proto "https://github.com/mitchellh/tree-sitter-proto" nil nil nil nil)
+                 (python "https://github.com/tree-sitter/tree-sitter-python" nil nil nil nil)
+                 (r "https://github.com/r-lib/tree-sitter-r" nil nil nil nil)
+                 (ruby "https://github.com/tree-sitter/tree-sitter-ruby" nil nil nil nil)
+                 (rust "https://github.com/tree-sitter/tree-sitter-rust" nil nil nil nil)
+                 (scala "https://github.com/tree-sitter/tree-sitter-scala" nil nil nil nil)
+                 (sql "https://github.com/DerekStride/tree-sitter-sql" "gh-pages" nil nil nil)
+                 (surface "https://github.com/connorlay/tree-sitter-surface" nil nil nil nil)
+                 (toml "https://github.com/tree-sitter/tree-sitter-toml" nil nil nil nil)
+                 (tsx "https://github.com/tree-sitter/tree-sitter-typescript" "master" "tsx/src" nil nil)
+                 (typescript "https://github.com/tree-sitter/tree-sitter-typescript" "master" "typescript/src" nil nil)
+                 (typst "https://github.com/uben0/tree-sitter-typst" "master" "src" nil nil)
+                 (verilog "https://github.com/gmlarumbe/tree-sitter-verilog" nil nil nil nil)
+                 (vhdl "https://github.com/alemuller/tree-sitter-vhdl" nil nil nil nil)
+                 (vue "https://github.com/tree-sitter-grammars/tree-sitter-vue" nil nil nil nil)
+                 (wast "https://github.com/wasm-lsp/tree-sitter-wasm" nil "wast/src" nil nil)
+                 (wat "https://github.com/wasm-lsp/tree-sitter-wasm" nil "wat/src" nil nil)
+                 (wgsl "https://github.com/mehmetoguzderin/tree-sitter-wgsl" nil nil nil nil)
+                 (yaml "https://github.com/tree-sitter-grammars/tree-sitter-yaml" nil nil nil nil)))
+    (cl-pushnew map treesit-language-source-alist :test #'eq :key #'car)))
 
 
-(use-package! evil-textobj-tree-sitter
+;; TODO: combobulate or evil-textobj-tree-sitter
-  :when (modulep! :editor evil +everywhere)
-  :defer t
-  :init (after! tree-sitter (require 'evil-textobj-tree-sitter))
-  :config
-  (defvar +tree-sitter-inner-text-objects-map (make-sparse-keymap))
-  (defvar +tree-sitter-outer-text-objects-map (make-sparse-keymap))
-  (defvar +tree-sitter-goto-previous-map (make-sparse-keymap))
-  (defvar +tree-sitter-goto-next-map (make-sparse-keymap))
 
-  (evil-define-key '(visual operator) 'tree-sitter-mode
-    "i" +tree-sitter-inner-text-objects-map
-    "a" +tree-sitter-outer-text-objects-map)
-  (evil-define-key 'normal 'tree-sitter-mode
-    "[g" +tree-sitter-goto-previous-map
-    "]g" +tree-sitter-goto-next-map)
 
-  (map! (:map +tree-sitter-inner-text-objects-map
+;; (use-package! combobulate
-         "A" (+tree-sitter-get-textobj '("parameter.inner" "call.inner"))
+;;   :commands combobulate-query-builder
-         "f" (+tree-sitter-get-textobj "function.inner")
+;;   :hook (prog-mode . combobulate-mode))
-         "F" (+tree-sitter-get-textobj "call.inner")
-         "C" (+tree-sitter-get-textobj "class.inner")
-         "v" (+tree-sitter-get-textobj "conditional.inner")
-         "l" (+tree-sitter-get-textobj "loop.inner"))
-        (:map +tree-sitter-outer-text-objects-map
-         "A" (+tree-sitter-get-textobj '("parameter.outer" "call.outer"))
-         "f" (+tree-sitter-get-textobj "function.outer")
-         "F" (+tree-sitter-get-textobj "call.outer")
-         "C" (+tree-sitter-get-textobj "class.outer")
-         "c" (+tree-sitter-get-textobj "comment.outer")
-         "v" (+tree-sitter-get-textobj "conditional.outer")
-         "l" (+tree-sitter-get-textobj "loop.outer"))
 
-        (:map +tree-sitter-goto-previous-map
-         "a" (+tree-sitter-goto-textobj "parameter.outer" t)
-         "f" (+tree-sitter-goto-textobj "function.outer" t)
-         "F" (+tree-sitter-goto-textobj "call.outer" t)
-         "C" (+tree-sitter-goto-textobj "class.outer" t)
-         "c" (+tree-sitter-goto-textobj "comment.outer" t)
-         "v" (+tree-sitter-goto-textobj "conditional.outer" t)
-         "l" (+tree-sitter-goto-textobj "loop.outer" t))
-        (:map +tree-sitter-goto-next-map
-         "a" (+tree-sitter-goto-textobj "parameter.outer")
-         "f" (+tree-sitter-goto-textobj "function.outer")
-         "F" (+tree-sitter-goto-textobj "call.outer")
-         "C" (+tree-sitter-goto-textobj "class.outer")
-         "c" (+tree-sitter-goto-textobj "comment.outer")
-         "v" (+tree-sitter-goto-textobj "conditional.outer")
-         "l" (+tree-sitter-goto-textobj "loop.outer")))
 
-  (after! which-key
+;; (use-package! evil-textobj-tree-sitter
-    (setq which-key-allow-multiple-replacements t)
+;;   :when (modulep! :editor evil +everywhere)
-    (pushnew!
+;;   :defer t
-     which-key-replacement-alist
+;;   :init (after! tree-sitter (require 'evil-textobj-tree-sitter))
-     '(("" . "\\`+?evil-textobj-tree-sitter-function--\\(.*\\)\\(?:.inner\\|.outer\\)") . (nil . "\\1")))))
+;;   :after-call doom-first-input-hook
+;;   :config
+;;   (defvar +tree-sitter-inner-text-objects-map (make-sparse-keymap))
+;;   (defvar +tree-sitter-outer-text-objects-map (make-sparse-keymap))
+;;   (defvar +tree-sitter-goto-previous-map (make-sparse-keymap))
+;;   (defvar +tree-sitter-goto-next-map (make-sparse-keymap))
+
+;;   (evil-define-key '(visual operator) 'tree-sitter-mode
+;;     "i" +tree-sitter-inner-text-objects-map
+;;     "a" +tree-sitter-outer-text-objects-map)
+;;   (evil-define-key 'normal 'tree-sitter-mode
+;;     "[g" +tree-sitter-goto-previous-map
+;;     "]g" +tree-sitter-goto-next-map)
+
+;;   (map! (:map +tree-sitter-inner-text-objects-map
+;;          "A" (+tree-sitter-get-textobj '("parameter.inner" "call.inner"))
+;;          "f" (+tree-sitter-get-textobj "function.inner")
+;;          "F" (+tree-sitter-get-textobj "call.inner")
+;;          "C" (+tree-sitter-get-textobj "class.inner")
+;;          "v" (+tree-sitter-get-textobj "conditional.inner")
+;;          "l" (+tree-sitter-get-textobj "loop.inner"))
+;;         (:map +tree-sitter-outer-text-objects-map
+;;          "A" (+tree-sitter-get-textobj '("parameter.outer" "call.outer"))
+;;          "f" (+tree-sitter-get-textobj "function.outer")
+;;          "F" (+tree-sitter-get-textobj "call.outer")
+;;          "C" (+tree-sitter-get-textobj "class.outer")
+;;          "c" (+tree-sitter-get-textobj "comment.outer")
+;;          "v" (+tree-sitter-get-textobj "conditional.outer")
+;;          "l" (+tree-sitter-get-textobj "loop.outer"))
+
+;;         (:map +tree-sitter-goto-previous-map
+;;          "a" (+tree-sitter-goto-textobj "parameter.outer" t)
+;;          "f" (+tree-sitter-goto-textobj "function.outer" t)
+;;          "F" (+tree-sitter-goto-textobj "call.outer" t)
+;;          "C" (+tree-sitter-goto-textobj "class.outer" t)
+;;          "c" (+tree-sitter-goto-textobj "comment.outer" t)
+;;          "v" (+tree-sitter-goto-textobj "conditional.outer" t)
+;;          "l" (+tree-sitter-goto-textobj "loop.outer" t))
+;;         (:map +tree-sitter-goto-next-map
+;;          "a" (+tree-sitter-goto-textobj "parameter.outer")
+;;          "f" (+tree-sitter-goto-textobj "function.outer")
+;;          "F" (+tree-sitter-goto-textobj "call.outer")
+;;          "C" (+tree-sitter-goto-textobj "class.outer")
+;;          "c" (+tree-sitter-goto-textobj "comment.outer")
+;;          "v" (+tree-sitter-goto-textobj "conditional.outer")
+;;          "l" (+tree-sitter-goto-textobj "loop.outer")))
+
+;;   (after! which-key
+;;     (setq which-key-allow-multiple-replacements t)
+;;     (pushnew!
+;;      which-key-replacement-alist
+;;      '(("" . "\\`+?evil-textobj-tree-sitter-function--\\(.*\\)\\(?:.inner\\|.outer\\)") . (nil . "\\1")))))

modules/tools/tree-sitter/doctor.el

@@ -1,4 +1,8 @@
-;;; tools/treesitter/doctor.el -*- lexical-binding: t; -*-
+;;; tools/tree-sitter/doctor.el -*- lexical-binding: t; -*-
 
 (unless (fboundp 'module-load)
-  (warn! "Emacs was not built with dynamic modules support. Tree sitter needs this to function"))
+  (warn! "Emacs was not built with dynamic modules support, which the treesit.el library requires"))
+
+(unless (and (fboundp 'treesit-available-p)
+             (treesit-available-p))
+  (error! "Treesit library not available. Did you build Emacs with tree-sitter support?"))

modules/tools/tree-sitter/packages.el

@@ -1,10 +1,18 @@
 ;; -*- no-byte-compile: t; -*-
 ;;; tools/tree-sitter/packages.el
 
-(package! tree-sitter :recipe (:branch "master") :pin "1c455b0953da06c40fcf1f21f1ac0c6e179b46d0")
+(package! treesit :built-in t)
-(package! tree-sitter-langs :pin "becd29c756a3272bc91d09de642df99a0fca6cee")
+(when (> emacs-major-version 28)
-(package! tree-sitter-indent :pin "4ef246db3e4ff99f672fe5e4b416c890f885c09e")
+  ;; (package! combobulate
-
+  ;;   :recipe '(;; If pulled from emacsmirror, this would otherwise pull in test
-(when (modulep! :editor evil +everywhere)
+  ;;             ;; repos that users don't need.
-  (package! evil-textobj-tree-sitter
+  ;;             :nonrecursive t
-    :pin "bce236e5d2cc2fa4eae7d284ffd19ad18d46349a"))
+  ;;             ;; HACK: This package has terrible autoload ettiquette, eagerly
+  ;;             ;;   loading a number of expensive packages at startup, so
+  ;;             ;;   autoloads are handled manually in config.el
+  ;;             :build (:not autoloads))
+  ;;   :pin "59b64d66d66eb84da6a2cedd152b1692378af674")
+  ;; (when (modulep! :editor evil +everywhere)
+  ;;   (package! evil-textobj-tree-sitter
+  ;;     :pin "bce236e5d2cc2fa4eae7d284ffd19ad18d46349a"))
+  )