feat: storage actions + git module actions

client/components/admin/admin-storage.vue

@@ -158,6 +158,24 @@
                       .caption.mt-3 Currently set to every #[strong {{getDefaultSchedule(tgt.syncInterval)}}].
                       .caption The default is every #[strong {{getDefaultSchedule(tgt.syncIntervalDefault)}}].
 
+                  template(v-if='tgt.actions && tgt.actions.length > 0')
+                    v-divider.mt-3
+                    v-subheader.pl-0 Actions
+                    v-container.pt-0(grid-list-xl, fluid)
+                      v-layout(row, wrap, fill-height)
+                        v-flex(xs12, lg6, xl4, v-for='act of tgt.actions')
+                          v-card.radius-7.grey(flat, :class='$vuetify.dark ? `darken-3-d5` : `lighten-3`', height='100%')
+                            v-card-text
+                              .subheading(v-html='act.label')
+                              .body-1.mt-2(v-html='act.hint')
+                              v-btn.mx-0.mt-3(
+                                @click='executeAction(tgt.key, act.handler)'
+                                outline
+                                :color='$vuetify.dark ? `blue` : `primary`'
+                                :disabled='runningAction'
+                                :loading='runningActionHandler === act.handler'
+                                ) Run
+
 </template>
 
 <script>
@@ -170,6 +188,7 @@ import { LoopingRhombusesSpinner } from 'epic-spinners'
 
 import statusQuery from 'gql/admin/storage/storage-query-status.gql'
 import targetsQuery from 'gql/admin/storage/storage-query-targets.gql'
+import targetExecuteActionMutation from 'gql/admin/storage/storage-mutation-executeaction.gql'
 import targetsSaveMutation from 'gql/admin/storage/storage-mutation-save-targets.gql'
 
 momentDurationFormatSetup(moment)
@@ -184,6 +203,8 @@ export default {
   },
   data() {
     return {
+      runningAction: false,
+      runningActionHandler: '',
       currentTab: 0,
       targets: [],
       status: []
@@ -209,12 +230,12 @@ export default {
         mutation: targetsSaveMutation,
         variables: {
           targets: this.targets.map(tgt => _.pick(tgt, [
-              'isEnabled',
-              'key',
-              'config',
-              'mode',
-              'syncInterval'
-            ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
+            'isEnabled',
+            'key',
+            'config',
+            'mode',
+            'syncInterval'
+          ])).map(str => ({...str, config: str.config.map(cfg => ({...cfg, value: JSON.stringify({ v: cfg.value.value })}))}))
         }
       })
       this.currentTab = 0
@@ -239,6 +260,30 @@ export default {
     },
     getDefaultSchedule(val) {
       return moment.duration(val).format('y [years], M [months], d [days], h [hours], m [minutes]')
+    },
+    async executeAction(targetKey, handler) {
+      this.$store.commit(`loadingStart`, 'admin-storage-executeaction')
+      this.runningAction = true
+      this.runningActionHandler = handler
+      try {
+        await this.$apollo.mutate({
+          mutation: targetExecuteActionMutation,
+          variables: {
+            targetKey,
+            handler
+          }
+        })
+        this.$store.commit('showNotification', {
+          message: 'Action completed.',
+          style: 'success',
+          icon: 'check'
+        })
+      } catch (err) {
+        console.warn(err)
+      }
+      this.runningAction = false
+      this.runningActionHandler = ''
+      this.$store.commit(`loadingStop`, 'admin-storage-executeaction')
     }
   },
   apollo: {

client/graph/admin/storage/storage-mutation-executeaction.gql

@@ -0,0 +1,12 @@
+mutation($targetKey: String!, $handler: String!) {
+  storage {
+    executeAction(targetKey: $targetKey, handler: $handler) {
+      responseResult {
+        succeeded
+        errorCode
+        slug
+        message
+      }
+    }
+  }
+}

client/graph/admin/storage/storage-query-targets.gql

@@ -17,6 +17,11 @@ query {
         key
         value
       }
+      actions {
+        handler
+        label
+        hint
+      }
     }
   }
 }

server/graph/resolvers/storage.js

@@ -77,6 +77,16 @@ module.exports = {
       } catch (err) {
         return graphHelper.generateError(err)
       }
+    },
+    async executeAction(obj, args, context) {
+      try {
+        await WIKI.models.storage.executeAction(args.targetKey, args.handler)
+        return {
+          responseResult: graphHelper.generateSuccess('Action completed.')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      }
     }
   }
 }

server/graph/schemas/storage.graphql

@@ -31,6 +31,11 @@ type StorageMutation {
   updateTargets(
     targets: [StorageTargetInput]!
   ): DefaultResponse @auth(requires: ["manage:system"])
+
+  executeAction(
+    targetKey: String!
+    handler: String!
+  ): DefaultResponse @auth(requires: ["manage:system"])
 }
 
 # -----------------------------------------------
@@ -51,6 +56,7 @@ type StorageTarget {
   syncInterval: String
   syncIntervalDefault: String
   config: [KeyValuePair]
+  actions: [StorageTargetAction]
 }
 
 input StorageTargetInput {
@@ -68,3 +74,9 @@ type StorageStatus {
   message: String!
   lastAttempt: String!
 }
+
+type StorageTargetAction {
+  handler: String!
+  label: String!
+  hint: String!
+}

server/helpers/page.js

@@ -36,5 +36,25 @@ module.exports = {
    */
   generateHash(opts) {
     return crypto.createHash('sha1').update(`${opts.locale}|${opts.path}|${opts.privateNS}`).digest('hex')
+  },
+  /**
+   * Inject Page Metadata
+   */
+  injectPageMetadata(page) {
+    let meta = [
+      ['title', page.title],
+      ['description', page.description],
+      ['published', page.isPublished.toString()],
+      ['date', page.updatedAt],
+      ['tags', '']
+    ]
+    switch (page.contentType) {
+      case 'markdown':
+        return '---\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n---\n\n' + page.content
+      case 'html':
+        return '<!--\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n-->\n\n' + page.content
+      default:
+        return page.content
+    }
   }
 }

server/models/pages.js

@@ -130,21 +130,7 @@ module.exports = class Page extends Model {
    * Inject page metadata into contents
    */
   injectMetadata () {
-    let meta = [
-      ['title', this.title],
-      ['description', this.description],
-      ['published', this.isPublished.toString()],
-      ['date', this.updatedAt],
-      ['tags', '']
-    ]
-    switch (this.contentType) {
-      case 'markdown':
-        return '---\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n---\n\n' + this.content
-      case 'html':
-        return '<!--\n' + meta.map(mt => `${mt[0]}: ${mt[1]}`).join('\n') + '\n-->\n\n' + this.content
-      default:
-        return this.content
-    }
+    return pageHelper.injectPageMetadata(this)
   }
 
   /**

server/models/storage.js

@@ -60,7 +60,7 @@ module.exports = class Storage extends Model {
           newTargets.push({
             key: target.key,
             isEnabled: false,
-            mode: target.defaultMode || 'push',
+            mode: target.defaultMode || 'push',
             syncInterval: target.schedule || 'P0D',
             config: _.transform(target.props, (result, value, key) => {
               _.set(result, key, value.default)
@@ -116,7 +116,7 @@ module.exports = class Storage extends Model {
       }
 
       // -> Initialize targets
-      for(let target of this.targets) {
+      for (let target of this.targets) {
         const targetDef = _.find(WIKI.data.storage, ['key', target.key])
         target.fn = require(`../modules/storage/${target.key}/storage`)
         target.fn.config = target.config
@@ -161,7 +161,7 @@ module.exports = class Storage extends Model {
 
   static async pageEvent({ event, page }) {
     try {
-      for(let target of this.targets) {
+      for (let target of this.targets) {
         await target.fn[event](page)
       }
     } catch (err) {
@@ -169,4 +169,22 @@ module.exports = class Storage extends Model {
       throw err
     }
   }
+
+  static async executeAction(targetKey, handler) {
+    try {
+      const target = _.find(this.targets, ['key', targetKey])
+      if (target) {
+        if (_.has(target.fn, handler)) {
+          await target.fn[handler]()
+        } else {
+          throw new Error('Invalid Handler for Storage Target')
+        }
+      } else {
+        throw new Error('Invalid or Inactive Storage Target')
+      }
+    } catch (err) {
+      WIKI.logger.warn(err)
+      throw err
+    }
+  }
 }

server/modules/storage/git/definition.yml

@@ -77,4 +77,10 @@ props:
 actions:
   - handler: syncUntracked
     label: Add Untracked Changes
-    hint: Output all content from the DB to the Git repo to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
+    hint: Output all content from the DB to the local Git repository to ensure all untracked content is saved. If you enabled Git after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing untracked changes.
+  - handler: sync
+    label: Force Sync
+    hint: Will trigger an immediate sync operation, regardless of the current sync schedule. The sync direction is respected.
+  - handler: importAll
+    label: Import Everything
+    hint: Will import all content currently in the local Git repository, regardless of the latest commit state. Useful for importing content from the remote repository created before git was enabled.

server/modules/storage/git/storage.js

@@ -2,6 +2,11 @@ const path = require('path')
 const sgit = require('simple-git/promise')
 const fs = require('fs-extra')
 const _ = require('lodash')
+const stream = require('stream')
+const Promise = require('bluebird')
+const pipeline = Promise.promisify(stream.pipeline)
+const klaw = require('klaw')
+const pageHelper = require('../../../helpers/page.js')
 
 const localeFolderRegex = /^([a-z]{2}(?:-[a-z]{2})?\/)?(.*)/i
 
@@ -154,66 +159,74 @@ module.exports = {
 
       const diff = await this.git.diffSummary(['-M', currentCommitLog.hash, latestCommitLog.hash])
       if (_.get(diff, 'files', []).length > 0) {
-        for (const item of diff.files) {
-          const contentType = getContenType(item.file)
-          if (!contentType) {
-            continue
-          }
-          const contentPath = getPagePath(item.file)
+        await this.processFiles(diff.files)
+      }
+    }
+  },
+  /**
+   * Process Files
+   *
+   * @param {Array<String>} files Array of files to process
+   */
+  async processFiles(files) {
+    for (const item of files) {
+      const contentType = getContenType(item.file)
+      if (!contentType) {
+        continue
+      }
+      const contentPath = getPagePath(item.file)
 
-          let itemContents = ''
-          try {
-            itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
-            const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
-            const currentPage = await WIKI.models.pages.query().findOne({
-              path: contentPath.path,
-              localeCode: contentPath.locale
-            })
-            if (currentPage) {
-              // Already in the DB, can mark as modified
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
-              await WIKI.models.pages.updatePage({
-                id: currentPage.id,
-                title: _.get(pageData, 'title', currentPage.title),
-                description: _.get(pageData, 'description', currentPage.description),
-                isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
-                isPrivate: false,
-                content: pageData.content,
-                authorId: 1,
-                skipStorage: true
-              })
-            } else {
-              // Not in the DB, can mark as new
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
-              const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
-              await WIKI.models.pages.createPage({
-                path: contentPath.path,
-                locale: contentPath.locale,
-                title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
-                description: _.get(pageData, 'description', ''),
-                isPublished: _.get(pageData, 'isPublished', true),
-                isPrivate: false,
-                content: pageData.content,
-                authorId: 1,
-                editor: pageEditor,
-                skipStorage: true
-              })
-            }
-          } catch (err) {
-            if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
-              // File was deleted by git, can safely mark as deleted in DB
-              WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
+      let itemContents = ''
+      try {
+        itemContents = await fs.readFile(path.join(this.repoPath, item.file), 'utf8')
+        const pageData = WIKI.models.pages.parseMetadata(itemContents, contentType)
+        const currentPage = await WIKI.models.pages.query().findOne({
+          path: contentPath.path,
+          localeCode: contentPath.locale
+        })
+        if (currentPage) {
+          // Already in the DB, can mark as modified
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as modified: ${item.file}`)
+          await WIKI.models.pages.updatePage({
+            id: currentPage.id,
+            title: _.get(pageData, 'title', currentPage.title),
+            description: _.get(pageData, 'description', currentPage.description),
+            isPublished: _.get(pageData, 'isPublished', currentPage.isPublished),
+            isPrivate: false,
+            content: pageData.content,
+            authorId: 1,
+            skipStorage: true
+          })
+        } else {
+          // Not in the DB, can mark as new
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as new: ${item.file}`)
+          const pageEditor = await WIKI.models.editors.getDefaultEditor(contentType)
+          await WIKI.models.pages.createPage({
+            path: contentPath.path,
+            locale: contentPath.locale,
+            title: _.get(pageData, 'title', _.last(contentPath.path.split('/'))),
+            description: _.get(pageData, 'description', ''),
+            isPublished: _.get(pageData, 'isPublished', true),
+            isPrivate: false,
+            content: pageData.content,
+            authorId: 1,
+            editor: pageEditor,
+            skipStorage: true
+          })
+        }
+      } catch (err) {
+        if (err.code === 'ENOENT' && item.deletions > 0 && item.insertions === 0) {
+          // File was deleted by git, can safely mark as deleted in DB
+          WIKI.logger.info(`(STORAGE/GIT) Page marked as deleted: ${item.file}`)
 
-              await WIKI.models.pages.deletePage({
-                path: contentPath.path,
-                locale: contentPath.locale,
-                skipStorage: true
-              })
-            } else {
-              WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
-              WIKI.logger.warn(err)
-            }
-          }
+          await WIKI.models.pages.deletePage({
+            path: contentPath.path,
+            locale: contentPath.locale,
+            skipStorage: true
+          })
+        } else {
+          WIKI.logger.warn(`(STORAGE/GIT) Failed to open ${item.file}`)
+          WIKI.logger.warn(err)
         }
       }
     }
@@ -278,5 +291,56 @@ module.exports = {
     await this.git.commit(`docs: rename ${page.sourcePath} to ${destinationFilePath}`, destinationFilePath, {
       '--author': `"${page.authorName} <${page.authorEmail}>"`
     })
+  },
+
+  /**
+   * HANDLERS
+   */
+  async importAll() {
+    WIKI.logger.info(`(STORAGE/GIT) Importing all content from local Git repo to the DB...`)
+    await pipeline(
+      klaw(this.repoPath, {
+        filter: (f) => {
+          return !_.includes(f, '.git')
+        }
+      }),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (file, enc, cb) => {
+          const relPath = file.path.substr(this.repoPath.length + 1)
+          if (relPath && relPath.length > 3) {
+            WIKI.logger.info(`(STORAGE/GIT) Processing ${relPath}...`)
+            await this.processFiles([{
+              file: relPath,
+              deletions: 0,
+              insertions: 0
+            }])
+          }
+          cb()
+        }
+      })
+    )
+    WIKI.logger.info('(STORAGE/GIT) Import completed.')
+  },
+  async syncUntracked() {
+    WIKI.logger.info(`(STORAGE/GIT) Adding all untracked content...`)
+    await pipeline(
+      WIKI.models.knex.column('path', 'localeCode', 'title', 'description', 'contentType', 'content', 'isPublished', 'updatedAt').select().from('pages').where({
+        isPrivate: false
+      }).stream(),
+      new stream.Transform({
+        objectMode: true,
+        transform: async (page, enc, cb) => {
+          const fileName = `${page.path}.${getFileExtension(page.contentType)}`
+          WIKI.logger.info(`(STORAGE/GIT) Adding ${fileName}...`)
+          const filePath = path.join(this.repoPath, fileName)
+          await fs.outputFile(filePath, pageHelper.injectPageMetadata(page), 'utf8')
+          await this.git.add(`./${fileName}`)
+          cb()
+        }
+      })
+    )
+    await this.git.commit(`docs: add all untracked content`)
+    WIKI.logger.info('(STORAGE/GIT) All content is now tracked.')
   }
 }