Removing algolia (#20633)

github · Aug 5, 2021 · 2dbea3737ad7e1d043da0c91a3ac30ce2d1f5cb4 · 2dbea37
1 parent 13caa0b
commit 2dbea3737ad7e1d043da0c91a3ac30ce2d1f5cb4
diff --git a/.env.example b/.env.example
@@ -1,3 +1 @@
-ALGOLIA_API_KEY=
-ALGOLIA_APPLICATION_ID=
 ALLOW_TRANSLATION_COMMITS=
diff --git a/.github/workflows/browser-test.yml b/.github/workflows/browser-test.yml
@@ -19,6 +19,10 @@ jobs:
       # Even if if doesn't do anything
       - name: Checkout
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+        with:
+          lfs: true
+      - name: Checkout LFS objects
+        run: git lfs checkout
 
       - name: Setup Node
         uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f

diff --git a/.github/workflows/dry-run-sync-search-indices.yml b/.github/workflows/dry-run-sync-search-indices.yml
@@ -26,7 +26,5 @@ jobs:
         run: npm run build
       - name: (Dry run) sync indices
         env:
-          ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
-          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: npm run sync-search-dry-run
diff --git a/.github/workflows/sync-search-indices.yml b/.github/workflows/sync-search-indices.yml
@@ -1,38 +1,95 @@
 name: Sync search indexes
 
-# **What it does**: This updates our search indexes after each deployment.
+# **What it does**: This workflow syncs the Lunr search indexes.
+#   The search indexes are checked into the lib/search/indexes directory.
+#   Search indexes are checked directly into the `main` branch on both the
+#   internal and open-source docs repositories. This workflow should be the
+#   only mechanism that the search indexes are modified. Because of that,
+#   repo-sync will not sync the search indexes because it should not detect
+#   a change.
 # **Why we have it**: We want our search indexes kept up to date.
 # **Who does it impact**: Anyone using search on docs.
 
+# **Testing: To test this workflow, use the workflow_dispatch event and trigger
+#   the workflow from the action tab. Select the branch with the changes to the
+#   workflow. Set `fetch-depth: 0` as an input to the checkout action to get all
+#   branches, including your test branch. Otherwise, you'll only get the main
+#   branch. For git lfs push and git push commands use the --dry-run switch to
+#   prevent pushes (e.g., git push --dry-run origin main --no-verify and
+#   git lfs push --dry-run public-docs-repo).
+#   The dry-run switch does everything but actually send the updates.
+
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
+  schedule:
+    - cron: '53 0/4 * * *' # Run every four hours at 53 minutes past the hour
 
 jobs:
-  updateIndices:
-    name: Update indices
+  updateIndexes:
+    name: Update indexes
     if: github.repository == 'github/docs-internal'
     runs-on: ubuntu-latest
     steps:
+      # Check out internal docs repository
       - name: checkout
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+        with:
+          token: ${{ secrets.DOCS_BOT }}
+
       - name: Setup Node
         uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f
         with:
           node-version: 16.x
           cache: npm
+
       - name: Install dependencies
         run: npm ci
+
       - name: Run build scripts
         run: npm run build
-      - name: sync indices
+
+      - name: Update search indexes
         env:
-          ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
-          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: npm run sync-search
+        # TODO remove version and language after first successful run to test
+        run: VERSION=github-ae@latest LANGUAGE=pt npm run sync-search
+
+      - name: Update private docs repository search indexes
+        # Git pre-push hooks push the LFS objects, so if you don't run them and
+        # don't push the LFS objects manually, the LFS objects won't get
+        # pushed. That will likely result in the push getting rejected.
+        # So if you don't use the pre-push hooks or you run with --no-verify
+        # the LFS objects need to be pushed first.
+        run: |
+          echo 'git checkout main'
+          git checkout main
+          echo 'git config user.name "GitHub Actions"'
+          git config user.name "GitHub Actions"
+          echo 'git config user.email action@github.com'
+          git config user.email action@github.com
+          echo 'git commit -am "update search indexes"'
+          git commit -am "update search indexes"
+          echo 'git lfs push origin'
+          git lfs push origin
+          echo 'git push origin main --no-verify'
+          git push origin main --no-verify
+
+      - name: Update open-source docs repository search indexes
+        # Git pre-push hooks push the LFS objects, so if you don't run them and
+        # don't push the LFS objects manually, the LFS objects won't get
+        # pushed. That will likely result in the push getting rejected.
+        # So if you don't use the pre-push hooks or you run with --no-verify
+        # the LFS objects need to be pushed first.
+        run: |
+          echo 'git remote add public-docs-repo https://github.com/github/docs.git'
+          git remote add public-docs-repo https://github.com/github/docs.git
+          echo 'git lfs push public-docs-repo'
+          git lfs push public-docs-repo
+          echo 'git pull public-docs-repo main'
+          git pull public-docs-repo main
+          echo 'git push public-docs-repo main --no-verify'
+          git push public-docs-repo main --no-verify
+
       - name: Send slack notification if workflow run fails
         uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
         if: failure()

diff --git a/.github/workflows/sync-single-english-index.yml b/.github/workflows/sync-single-english-index.yml
@@ -5,6 +5,7 @@ name: Sync Single English Index
 # **Who does it impact**:
 
 on:
+  workflow_dispatch:
   pull_request:
     types:
       - labeled
@@ -39,9 +40,15 @@ jobs:
         env:
           VERSION: ${{ steps.getVersion.outputs.versionToSync }}
           LANGUAGE: 'en'
-          ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
-          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           npm run build
           npm run sync-search
+      - name: Check in search index
+        uses: EndBug/add-and-commit@2bdc0a61a03738a1d1bda24d566ad0dbe3083d87
+        with:
+          # The arguments for the `git add` command
+          add: 'lib/search/indexes'
+
+          # The message for the commit
+          message: 'Updated search index for ${{ steps.getVersion.outputs.versionToSync }}'
diff --git a/.github/workflows/triage-unallowed-contributions.yml b/.github/workflows/triage-unallowed-contributions.yml
@@ -16,6 +16,7 @@ on:
       - 'lib/redirects/**'
       - 'lib/rest/**'
       - 'lib/webhooks/**'
+      - 'lib/search/indexes/**'
       - 'scripts/**'
       - 'translations/**'
       - 'package*.json'
@@ -76,6 +77,7 @@ jobs:
               - 'lib/redirects/**'
               - 'lib/rest/**'
               - 'lib/webhooks/**'
+              - 'lib/search/indexes/**'
               - 'scripts/**'
               - 'translations/**'
               - 'package*.json'
@@ -101,6 +103,7 @@ jobs:
               'lib/redirects/**',
               'lib/rest/**',
               'lib/webhooks/**',
+              'lib/search/indexes/**'
               'scripts/**',
               'translations/**',
               'package*.json',

diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,3 @@
-.algolia-cache
 .search-cache
 .DS_Store
 .env

diff --git a/components/Search.tsx b/components/Search.tsx
@@ -41,7 +41,7 @@ export function Search({ isStandalone = false, updateSearchParams = true, childr
 
   // If the user shows up with a query in the URL, go ahead and search for it
   useEffect(() => {
-    if (router.query.query) {
+    if (updateSearchParams && router.query.query) {
       /* await */ fetchSearchResults((router.query.query as string).trim())
     }
   }, [])

diff --git a/components/page-header/Header.tsx b/components/page-header/Header.tsx
@@ -48,7 +48,7 @@ export const Header = () => {
           {/* <!-- GitHub.com homepage and 404 page has a stylized search; Enterprise homepages do not --> */}
           {relativePath !== 'index.md' && error !== '404' && (
             <div className="d-inline-block ml-4">
-              <Search />
+              {router.asPath !== '/graphql/overview/explorer' && <Search />}
             </div>
           )}
         </div>
@@ -112,7 +112,7 @@ export const Header = () => {
               {/* <!-- GitHub.com homepage and 404 page has a stylized search; Enterprise homepages do not --> */}
               {relativePath !== 'index.md' && error !== '404' && (
                 <div className="pt-3 border-top">
-                  <Search />
+                  {router.asPath !== '/graphql/overview/explorer' && <Search />}
                 </div>
               )}
             </div>

diff --git a/contributing/search.md b/contributing/search.md
@@ -37,17 +37,19 @@ Using the attribute `topics` in your query will only return results that have th
 
 ## Production deploys
 
-A [GitHub Actions workflow](.github/workflows/sync-search-indices.yml) triggered by pushes to the `main` branch syncs the search data. This process generates structured data for all pages on the site, compares that data to what's currently on search, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations.
+A [GitHub Actions workflow](.github/workflows/sync-search-indices.yml) that runs every four hours syncs the search data. This process generates structured data for all pages on the site, compares that data to what's currently on search, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations.
 
 The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3Asearch) of the repo.
 
-Because the workflow runs after a branch is merged to `main`, there is a slight delay for search data updates to appear on the site.
+## Manually triggering the search index update workflow
 
-## Manual sync from a checkout
+You can manually run the workflow to generate the indexes after you push your changes to `main` to speed up the indexing when needed. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml).
 
-It is also possible to manually sync the indices from your local checkout of the repo, before your branch is merged to `main`.
+## Generating search indexes for your local checkout
 
-**Prerequisite:** Make sure the environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. You can find these values on [Algolia](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). _Remove this paragraph when we switch to Lunr._
+You can locally generate search indexes, but please do not check them into your local branch because they can get out-of-sync with the `main` branch quickly.
+
+To locally generate the English version of the Dotcom search index locally, run `LANGUAGE=en VERSION=free-pro-team@latest npm run sync-search`. See [Build and sync](#build-and-sync) below for more details. To revert those files run `git checkout lib/search/indexes`.
 
 ### Build without sync (dry run)
 
@@ -75,7 +77,7 @@ VERSION=<PLAN@RELEASE LANGUAGE=<TWO-LETTER CODE> npm run sync-search
 ```
 You can set `VERSION` and `LANGUAGE` individually, too.
 
-Substitute a currently supported version for `<PLAN@RELEASE>` and a currently supported two-letter language code for `<TWO-LETTER-CODE>`.
+Substitute a currently supported version for `<PLAN@RELEASE>` and a currently supported two-letter language code for `<TWO-LETTER-CODE>`. Languages and versions are lowercase. The options for version are currently `free-pro-team`, `github-ae`, and `enterprise-server`.
 
 ## Label-triggered Actions workflow
 
@@ -95,7 +97,7 @@ Why do we need this? For our daily shipping needs, it's tolerable that search up
 
 ### Actions workflow files
 
-- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys).
+- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices on the `main` branch every four hours. Search indices are committed directly to the `main` branch on both the `github/docs-internal` and `github/docs` repositories. It can also be run manually. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml).
 - [`.github/workflows/dry-run-sync-search-indices.yml`](.github/workflows/dry-run-sync-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out.
 - [`.github/workflows/sync-single-english-index.yml`](.github/workflows/sync-single-english-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered Actions workflow](#label-triggered-actions-workflow)" for details.
 

diff --git a/lib/search/algolia-search.js b/lib/search/algolia-search.js
diff --git a/middleware/csp.js b/middleware/csp.js
@@ -12,7 +12,7 @@ export default function csp(req, res, next) {
   const csp = {
     directives: {
       defaultSrc: ["'none'"],
-      connectSrc: ["'self'", '*.algolia.net', '*.algolianet.com'],
+      connectSrc: ["'self'"],
       fontSrc: ["'self'", 'data:', AZURE_STORAGE_URL],
       imgSrc: [
         "'self'",

diff --git a/middleware/search.js b/middleware/search.js
@@ -2,7 +2,6 @@ import express from 'express'
 import libLanguages from '../lib/languages.js'
 import searchVersions from '../lib/search/versions.js'
 import loadLunrResults from '../lib/search/lunr-search.js'
-import loadAlgoliaResults from '../lib/search/algolia-search.js'
 const languages = new Set(Object.keys(libLanguages))
 const versions = new Set(Object.values(searchVersions))
 
@@ -24,11 +23,12 @@ router.get('/', async function postSearch(req, res, next) {
   }
 
   try {
-    const results =
-      process.env.AIRGAP || req.cookies.AIRGAP
-        ? await loadLunrResults({ version, language, query: `${query} ${filters || ''}`, limit })
-        : await loadAlgoliaResults({ version, language, query, filters, limit })
-
+    const results = await loadLunrResults({
+      version,
+      language,
+      query: `${query} ${filters || ''}`,
+      limit,
+    })
     // Only reply if the headers have not been sent and the request was not aborted...
     if (!res.headersSent && !req.aborted) {
       return res.status(200).json(results)