Move In Progress ostrio-files changes to separate branch, and revert ostrio-files changes, so that:

- Export to CSV/TSV with custom fields works
- Attachments are not exported to disk
- It is possible to build arm64/s390x versions again.

Thanks to xet7 !

Related #3110

.meteor/packages

@@ -98,4 +98,3 @@ percolate:synced-cron
 easylogic:summernote
 cfs:filesystem
 ostrio:cookies
-ostrio:files

.meteor/versions

@@ -134,7 +134,6 @@ observe-sequence@1.0.16
 ongoworks:speakingurl@1.1.0
 ordered-dict@1.1.0
 ostrio:cookies@2.6.0
-ostrio:files@1.14.2
 peerlibrary:assert@0.3.0
 peerlibrary:base-component@0.16.0
 peerlibrary:blaze-components@0.15.1

client/components/activities/activities.js

@@ -151,23 +151,21 @@ BlazeComponent.extendComponent({
   },
 
   attachmentLink() {
-    const activity = this.currentData().activity;
-    const attachment = activity.attachment();
-    const link = attachment ? attachment.link('original', '/') : null;
+    const attachment = this.currentData().activity.attachment();
     // trying to display url before file is stored generates js errors
     return (
       (attachment &&
-        link &&
+        attachment.url({ download: true }) &&
         Blaze.toHTML(
           HTML.A(
             {
-              href: link,
+              href: attachment.url({ download: true }),
               target: '_blank',
             },
-            attachment.name,
+            attachment.name(),
           ),
         )) ||
-      activity.attachmentName
+      this.currentData().activity.attachmentName
     );
   },
 

client/components/cards/attachments.jade

@@ -18,19 +18,12 @@ template(name="attachmentDeletePopup")
   p {{_ "attachment-delete-pop"}}
   button.js-confirm.negate.full(type="submit") {{_ 'delete'}}
 
-template(name="uploadingPopup")
-  .uploading-info
-    span.upload-percentage {{progress}}%
-    .upload-progress-frame
-      .upload-progress-bar(style="width: {{progress}}%;")
-    span.upload-size {{fileSize}}
-
 template(name="attachmentsGalery")
   .attachments-galery
     each attachments
       .attachment-item
-        a.attachment-thumbnail.swipebox(href="{{url}}" download="{{name}}" title="{{name}}")
-          if isUploaded 
+        a.attachment-thumbnail.swipebox(href="{{url}}" title="{{name}}")
+          if isUploaded
             if isImage
               img.attachment-thumbnail-img(src="{{url}}")
             else
@@ -40,7 +33,7 @@ template(name="attachmentsGalery")
         p.attachment-details
           = name
           span.attachment-details-actions
-            a.js-download(href="{{url download=true}}" download="{{name}}")
+            a.js-download(href="{{url download=true}}")
               i.fa.fa-download
               | {{_ 'download'}}
             if currentUser.isBoardMember

client/components/cards/attachments.js

@@ -13,10 +13,10 @@ Template.attachmentsGalery.events({
     event.stopPropagation();
   },
   'click .js-add-cover'() {
-    Cards.findOne(this.meta.cardId).setCover(this._id);
+    Cards.findOne(this.cardId).setCover(this._id);
   },
   'click .js-remove-cover'() {
-    Cards.findOne(this.meta.cardId).unsetCover();
+    Cards.findOne(this.cardId).unsetCover();
   },
   'click .js-preview-image'(event) {
     Popup.open('previewAttachedImage').call(this, event);
@@ -45,63 +45,22 @@ Template.attachmentsGalery.events({
   },
 });
 
-Template.attachmentsGalery.helpers({
-  url() {
-    return Attachments.link(this, 'original', '/'); 
-  },
-  isUploaded() {
-    return !this.meta.uploading;
-  },
-  isImage() {
-    return !!this.isImage;
-  },
-});
-
 Template.previewAttachedImagePopup.events({
   'click .js-large-image-clicked'() {
     Popup.close();
   },
 });
 
-Template.previewAttachedImagePopup.helpers({
-  url() {
-    return Attachments.link(this, 'original', '/');
-  }
-});
-
-// For uploading popup
-
-let uploadFileSize = new ReactiveVar('');
-let uploadProgress = new ReactiveVar(0);
-
 Template.cardAttachmentsPopup.events({
-  'change .js-attach-file'(event, instance) {
+  'change .js-attach-file'(event) {
     const card = this;
-    const callbacks = {
-		    onBeforeUpload: (err, fileData) => {
-          Popup.open('uploading')(this.clickEvent);
-          uploadFileSize.set('...');
-          uploadProgress.set(0);
-          return true;
-        },
-        onUploaded: (err, attachment) => {
-          if (attachment && attachment._id && attachment.isImage) {
-            card.setCover(attachment._id);
-          }
-          Popup.close();
-        },
-        onStart: (error, fileData) => {
-          uploadFileSize.set(formatBytes(fileData.size));
-        },
-				onError: (err, fileObj) => {
-          console.log('Error!', err);
-        },
-        onProgress: (progress, fileData) => {
-          uploadProgress.set(progress);
-        }
-    };
     const processFile = f => {
-      Utils.processUploadedAttachment(card, f, callbacks);
+      Utils.processUploadedAttachment(card, f, attachment => {
+        if (attachment && attachment._id && attachment.isImage()) {
+          card.setCover(attachment._id);
+        }
+        Popup.close();
+      });
     };
 
     FS.Utility.eachFile(event, f => {
@@ -141,22 +100,12 @@ Template.cardAttachmentsPopup.events({
     });
   },
   'click .js-computer-upload'(event, templateInstance) {
-    this.clickEvent = event;
     templateInstance.find('.js-attach-file').click();
     event.preventDefault();
   },
   'click .js-upload-clipboard-image': Popup.open('previewClipboardImage'),
 });
 
-Template.uploadingPopup.helpers({
-  fileSize: () => {
-    return uploadFileSize.get();
-  },
-  progress: () => {
-    return uploadProgress.get();
-  }
-});
-
 const MAX_IMAGE_PIXEL = Utils.MAX_IMAGE_PIXEL;
 const COMPRESS_RATIO = Utils.IMAGE_COMPRESS_RATIO;
 let pastedResults = null;
@@ -200,26 +149,20 @@ Template.previewClipboardImagePopup.events({
     if (results && results.file) {
       window.oPasted = pastedResults;
       const card = this;
-      const settings = {
-        file: results.file,
-        streams: 'dynamic',
-        chunkSize: 'dynamic',
-      };
+      const file = new FS.File(results.file);
       if (!results.name) {
         // if no filename, it's from clipboard. then we give it a name, with ext name from MIME type
         if (typeof results.file.type === 'string') {
-          settings.fileName =
-            new Date().getTime() + results.file.type.replace('.+/', '');
+          file.name(results.file.type.replace('image/', 'clipboard.'));
         }
       }
-      settings.meta = {};
-      settings.meta.updatedAt = new Date().getTime();
-      settings.meta.boardId = card.boardId;
-      settings.meta.cardId = card._id;
-      settings.meta.userId = Meteor.userId();
-      const attachment = Attachments.insert(settings);
+      file.updatedAt(new Date());
+      file.boardId = card.boardId;
+      file.cardId = card._id;
+      file.userId = Meteor.userId();
+      const attachment = Attachments.insert(file);
 
-      if (attachment && attachment._id && attachment.isImage) {
+      if (attachment && attachment._id && attachment.isImage()) {
         card.setCover(attachment._id);
       }
 
@@ -229,15 +172,3 @@ Template.previewClipboardImagePopup.events({
     }
   },
 });
-
-function formatBytes(bytes, decimals = 2) {
-    if (bytes === 0) return '0 Bytes';
-
-    const k = 1024;
-    const dm = decimals < 0 ? 0 : decimals;
-    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
-
-    const i = Math.floor(Math.log(bytes) / Math.log(k));
-
-    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
-}

client/components/cards/attachments.styl

@@ -64,17 +64,6 @@
   border: 1px solid black
   box-shadow: 0 1px 2px rgba(0,0,0,.2)
 
-.uploading-info
-  .upload-progress-frame
-    background-color: grey;
-    border: 1px solid;
-    height: 22px;
-
-  .upload-progress-bar
-    background-color: blue;
-    height: 20px;
-    padding: 1px;
-
 @media screen and (max-width: 800px)
   .attachments-galery
     flex-direction

client/components/cards/minicard.jade

@@ -11,7 +11,7 @@ template(name="minicard")
         .handle
           .fa.fa-arrows
     if cover
-      .minicard-cover(style="background-image: url('{{coverUrl}}');")
+      .minicard-cover(style="background-image: url('{{cover.url}}');")
     if labels
       .minicard-labels
         each labels

client/components/cards/minicard.js

@@ -52,7 +52,4 @@ Template.minicard.helpers({
       return false;
     }
   },
-  coverUrl() {
-    return Attachments.findOne(this.coverId).link('original', '/');
-  },
 });

client/components/main/editor.js

@@ -152,31 +152,33 @@ Template.editor.onRendered(() => {
                 const processData = function(fileObj) {
                   Utils.processUploadedAttachment(
                     currentCard,
-                    fileObj, 
-                    { onUploaded:
-                      attachment => {
-                        if (attachment && attachment._id && attachment.isImage) {
-                          attachment.one('uploaded', function() {
-                            const maxTry = 3;
-                            const checkItvl = 500;
-                            let retry = 0;
-                            const checkUrl = function() {
-                              // even though uploaded event fired, attachment.url() is still null somehow //TODO
-                              const url = Attachments.link(attachment, 'original', '/');
-                              if (url) {
-                                insertImage(
-                                  `${location.protocol}//${location.host}${url}`,
-                                );
-                              } else {
-                                retry++;
-                                if (retry < maxTry) {
-                                  setTimeout(checkUrl, checkItvl);
-                                }
+                    fileObj,
+                    attachment => {
+                      if (
+                        attachment &&
+                        attachment._id &&
+                        attachment.isImage()
+                      ) {
+                        attachment.one('uploaded', function() {
+                          const maxTry = 3;
+                          const checkItvl = 500;
+                          let retry = 0;
+                          const checkUrl = function() {
+                            // even though uploaded event fired, attachment.url() is still null somehow //TODO
+                            const url = attachment.url();
+                            if (url) {
+                              insertImage(
+                                `${location.protocol}//${location.host}${url}`,
+                              );
+                            } else {
+                              retry++;
+                              if (retry < maxTry) {
+                                setTimeout(checkUrl, checkItvl);
                               }
-                            };
-                            checkUrl();
-                          });
-                        }
+                            }
+                          };
+                          checkUrl();
+                        });
                       }
                     },
                   );

client/lib/utils.js

@@ -61,38 +61,30 @@ Utils = {
   },
   MAX_IMAGE_PIXEL: Meteor.settings.public.MAX_IMAGE_PIXEL,
   COMPRESS_RATIO: Meteor.settings.public.IMAGE_COMPRESS_RATIO,
-  processUploadedAttachment(card, fileObj, callbacks) {
+  processUploadedAttachment(card, fileObj, callback) {
+    const next = attachment => {
+      if (typeof callback === 'function') {
+        callback(attachment);
+      }
+    };
     if (!card) {
-      return onUploaded();
+      return next();
     }
-    let settings = {
-      file: fileObj,
-      streams: 'dynamic',
-      chunkSize: 'dynamic',
-    };
-    settings.meta = {
-      uploading: true
-    };
+    const file = new FS.File(fileObj);
     if (card.isLinkedCard()) {
-      settings.meta.boardId = Cards.findOne(card.linkedId).boardId;
-      settings.meta.cardId = card.linkedId;
+      file.boardId = Cards.findOne(card.linkedId).boardId;
+      file.cardId = card.linkedId;
     } else {
-      settings.meta.boardId = card.boardId;
-      settings.meta.swimlaneId = card.swimlaneId;
-      settings.meta.listId = card.listId;
-      settings.meta.cardId = card._id;
+      file.boardId = card.boardId;
+      file.swimlaneId = card.swimlaneId;
+      file.listId = card.listId;
+      file.cardId = card._id;
     }
-    settings.meta.userId = Meteor.userId();
-    if (typeof callbacks === 'function') {
-      settings.onEnd = callbacks;
-    } else {
-      for (const key in callbacks) {
-        if (key.substring(0, 2) === 'on') {
-          settings[key] = callbacks[key];
-        }
-      }
+    file.userId = Meteor.userId();
+    if (file.original) {
+      file.original.name = fileObj.name;
     }
-    Attachments.insert(settings);
+    return next(Attachments.insert(file));
   },
   shrinkImage(options) {
     // shrink image to certain size

models/activities.js

@@ -217,7 +217,7 @@ if (Meteor.isServer) {
     }
     if (activity.attachmentId) {
       const attachment = activity.attachment();
-      params.attachment = attachment.name;
+      params.attachment = attachment.original.name;
       params.attachmentId = attachment._id;
     }
     if (activity.checklistId) {

models/attachments.js

@@ -1,127 +1,263 @@
-import { FilesCollection } from 'meteor/ostrio:files';
-const fs = require('fs');
-
-const collectionName = 'attachments2';
-
-Attachments = new FilesCollection({
-  storagePath: storagePath(),
-  debug: false, 
-//  allowClientCode: true,
-  collectionName: 'attachments2',
-  onAfterUpload: onAttachmentUploaded,
-  onBeforeRemove: onAttachmentRemoving
-});
+const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
+const storeName = 'attachments';
+const defaultStoreOptions = {
+  beforeWrite: fileObj => {
+    if (!fileObj.isImage()) {
+      return {
+        type: 'application/octet-stream',
+      };
+    }
+    return {};
+  },
+};
+let store;
+if (localFSStore) {
+  // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
+  const fs = Npm.require('fs');
+  const path = Npm.require('path');
+  const mongodb = Npm.require('mongodb');
+  const Grid = Npm.require('gridfs-stream');
+  // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
+  let pathname = localFSStore;
+  /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
 
-if (Meteor.isServer) {
-  Meteor.startup(() => {
-    Attachments.collection._ensureIndex({ cardId: 1 });
-  });
+  if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
+    pathname = path.join(
+      __meteor_bootstrap__.serverDir,
+      `../../../cfs/files/${storeName}`,
+    );
+  }
 
-  // TODO: Permission related
-  Attachments.allow({
-    insert() {
-      return false;
-    },
-    update() {
-      return true;
-    },
-    remove() {
-      return true;
+  if (!pathname)
+    throw new Error('FS.Store.FileSystem unable to determine path');
+
+  // Check if we have '~/foo/bar'
+  if (pathname.split(path.sep)[0] === '~') {
+    const homepath =
+      process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
+    if (homepath) {
+      pathname = pathname.replace('~', homepath);
+    } else {
+      throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
     }
-  });
+  }
 
-  Meteor.methods({
-    cloneAttachment(file, overrides) {
-      check(file, Object);
-      check(overrides, Match.Maybe(Object));
-      const path = file.path;
-      const opts = {
-          fileName: file.name,
-          type: file.type,
-          meta: file.meta,
-          userId: file.userId
+  // Set absolute path
+  const absolutePath = path.resolve(pathname);
+
+  const _FStore = new FS.Store.FileSystem(storeName, {
+    path: localFSStore,
+    ...defaultStoreOptions,
+  });
+  const GStore = {
+    fileKey(fileObj) {
+      const key = {
+        _id: null,
+        filename: null,
       };
-      for (let key in overrides) {
-        if (key === 'meta') {
-          for (let metaKey in overrides.meta) {
-            opts.meta[metaKey] = overrides.meta[metaKey];
-          }
-        } else {
-          opts[key] = overrides[key];
-        }
+
+      // If we're passed a fileObj, we retrieve the _id and filename from it.
+      if (fileObj) {
+        const info = fileObj._getInfo(storeName, {
+          updateFileRecordFirst: false,
+        });
+        key._id = info.key || null;
+        key.filename =
+          info.name ||
+          fileObj.name({ updateFileRecordFirst: false }) ||
+          `${fileObj.collectionName}-${fileObj._id}`;
       }
-      const buffer = fs.readFileSync(path);
-      Attachments.write(buffer, opts, (err, fileRef) => {
+
+      // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
+      return key;
+    },
+    db: undefined,
+    mongoOptions: { useNewUrlParser: true },
+    mongoUrl: process.env.MONGO_URL,
+    init() {
+      this._init(err => {
+        this.inited = !err;
+      });
+    },
+    _init(callback) {
+      const self = this;
+      mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
+        err,
+        db,
+      ) {
         if (err) {
-          console.log('Error when cloning record', err);
+          return callback(err);
         }
+        self.db = db;
+        return callback(null);
       });
-      return true;
+      return;
+    },
+    createReadStream(fileKey, options) {
+      const self = this;
+      if (!self.inited) {
+        self.init();
+        return undefined;
+      }
+      options = options || {};
+
+      // Init GridFS
+      const gfs = new Grid(self.db, mongodb);
+
+      // Set the default streamning settings
+      const settings = {
+        _id: new mongodb.ObjectID(fileKey._id),
+        root: `cfs_gridfs.${storeName}`,
+      };
+
+      // Check if this should be a partial read
+      if (
+        typeof options.start !== 'undefined' &&
+        typeof options.end !== 'undefined'
+      ) {
+        // Add partial info
+        settings.range = {
+          startPos: options.start,
+          endPos: options.end,
+        };
+      }
+      return gfs.createReadStream(settings);
+    },
+  };
+  GStore.init();
+  const CRS = 'createReadStream';
+  const _CRS = `_${CRS}`;
+  const FStore = _FStore._transform;
+  FStore[_CRS] = FStore[CRS].bind(FStore);
+  FStore[CRS] = function(fileObj, options) {
+    let stream;
+    try {
+      const localFile = path.join(
+        absolutePath,
+        FStore.storage.fileKey(fileObj),
+      );
+      const state = fs.statSync(localFile);
+      if (state) {
+        stream = FStore[_CRS](fileObj, options);
+      }
+    } catch (e) {
+      // file is not there, try GridFS ?
+      stream = undefined;
     }
+    if (stream) return stream;
+    else {
+      try {
+        const stream = GStore[CRS](GStore.fileKey(fileObj), options);
+        return stream;
+      } catch (e) {
+        return undefined;
+      }
+    }
+  }.bind(FStore);
+  store = _FStore;
+} else {
+  store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
+    // XXX Add a new store for cover thumbnails so we don't load big images in
+    // the general board view
+    // If the uploaded document is not an image we need to enforce browser
+    // download instead of execution. This is particularly important for HTML
+    // files that the browser will just execute if we don't serve them with the
+    // appropriate `application/octet-stream` MIME header which can lead to user
+    // data leaks. I imagine other formats (like PDF) can also be attack vectors.
+    // See https://github.com/wekan/wekan/issues/99
+    // XXX Should we use `beforeWrite` option of CollectionFS instead of
+    // collection-hooks?
+    // We should use `beforeWrite`.
+    ...defaultStoreOptions,
   });
+}
+Attachments = new FS.Collection('attachments', {
+  stores: [store],
+});
 
-  Meteor.publish(collectionName, function() {
-    return Attachments.find().cursor;
+if (Meteor.isServer) {
+  Meteor.startup(() => {
+    Attachments.files._ensureIndex({ cardId: 1 });
   });
-} else {
-  Meteor.subscribe(collectionName);
-}
 
-function storagePath(defaultPath) {
-  const storePath = process.env.ATTACHMENTS_STORE_PATH;
-  return storePath ? storePath : defaultPath;
+  Attachments.allow({
+    insert(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    update(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    remove(userId, doc) {
+      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
+    },
+    // We authorize the attachment download either:
+    // - if the board is public, everyone (even unconnected) can download it
+    // - if the board is private, only board members can download it
+    download(userId, doc) {
+      const board = Boards.findOne(doc.boardId);
+      if (board.isPublic()) {
+        return true;
+      } else {
+        return board.hasMember(userId);
+      }
+    },
+
+    fetch: ['boardId'],
+  });
 }
 
-function onAttachmentUploaded(fileRef) {
-  Attachments.update({_id:fileRef._id}, {$set: {"meta.uploading": false}});
-  if (!fileRef.meta.source || fileRef.meta.source !== 'import') {
-    // Add activity about adding the attachment
+// XXX Enforce a schema for the Attachments CollectionFS
+
+if (Meteor.isServer) {
+  Attachments.files.after.insert((userId, doc) => {
+    // If the attachment doesn't have a source field
+    // or its source is different than import
+    if (!doc.source || doc.source !== 'import') {
+      // Add activity about adding the attachment
+      Activities.insert({
+        userId,
+        type: 'card',
+        activityType: 'addAttachment',
+        attachmentId: doc._id,
+        // this preserves the name so that notifications can be meaningful after
+        // this file is removed
+        attachmentName: doc.original.name,
+        boardId: doc.boardId,
+        cardId: doc.cardId,
+        listId: doc.listId,
+        swimlaneId: doc.swimlaneId,
+      });
+    } else {
+      // Don't add activity about adding the attachment as the activity
+      // be imported and delete source field
+      Attachments.update(
+        {
+          _id: doc._id,
+        },
+        {
+          $unset: {
+            source: '',
+          },
+        },
+      );
+    }
+  });
+
+  Attachments.files.before.remove((userId, doc) => {
     Activities.insert({
-      userId: fileRef.userId,
+      userId,
       type: 'card',
-      activityType: 'addAttachment',
-      attachmentId: fileRef._id,
+      activityType: 'deleteAttachment',
+      attachmentId: doc._id,
       // this preserves the name so that notifications can be meaningful after
-      // this file is removed 
-      attachmentName: fileRef.name,
-      boardId: fileRef.meta.boardId,
-      cardId: fileRef.meta.cardId,
-      listId: fileRef.meta.listId,
-      swimlaneId: fileRef.meta.swimlaneId,
+      // this file is removed
+      attachmentName: doc.original.name,
+      boardId: doc.boardId,
+      cardId: doc.cardId,
+      listId: doc.listId,
+      swimlaneId: doc.swimlaneId,
     });
-  } else {
-    // Don't add activity about adding the attachment as the activity
-    // be imported and delete source field
-    Attachments.collection.update(
-      {
-        _id: fileRef._id,
-      },
-      {
-        $unset: {
-          'meta.source': '',
-        },
-      },
-    );
-  }
-}
-
-function onAttachmentRemoving(cursor) {
-  const file = cursor.get()[0];
-  const meta = file.meta;
-  Activities.insert({
-    userId: this.userId,
-    type: 'card',
-    activityType: 'deleteAttachment',
-    attachmentId: file._id,
-    // this preserves the name so that notifications can be meaningful after
-    // this file is removed
-    attachmentName: file.name,
-    boardId: meta.boardId,
-    cardId: meta.cardId,
-    listId: meta.listId,
-    swimlaneId: meta.swimlaneId,
   });
-  return true;
 }
 
 export default Attachments;

models/cards.js

@@ -412,14 +412,10 @@ Cards.helpers({
     const _id = Cards.insert(this);
 
     // Copy attachments
-    oldCard.attachments().forEach((file) => {
-      Meteor.call('cloneAttachment', file, 
-        {
-          meta: {
-            cardId: _id
-          }
-        }
-      );
+    oldCard.attachments().forEach(att => {
+      att.cardId = _id;
+      delete att._id;
+      return Attachments.insert(att);
     });
 
     // copy checklists
@@ -522,15 +518,14 @@ Cards.helpers({
   attachments() {
     if (this.isLinkedCard()) {
       return Attachments.find(
-        { 'meta.cardId': this.linkedId },
+        { cardId: this.linkedId },
         { sort: { uploadedAt: -1 } },
       );
     } else {
-      let ret = Attachments.find(
-        { 'meta.cardId': this._id },
+      return Attachments.find(
+        { cardId: this._id },
         { sort: { uploadedAt: -1 } },
       );
-      return ret;
     }
   },
 
@@ -539,7 +534,7 @@ Cards.helpers({
     const cover = Attachments.findOne(this.coverId);
     // if we return a cover before it is fully stored, we will get errors when we try to display it
     // todo XXX we could return a default "upload pending" image in the meantime?
-    return cover && cover.link();
+    return cover && cover.url() && cover;
   },
 
   checklists() {

models/export.js

@@ -1,3 +1,4 @@
+import { Exporter } from './exporter';
 /* global JsonRoutes */
 if (Meteor.isServer) {
   // todo XXX once we have a real API in place, move that route there
@@ -7,10 +8,10 @@ if (Meteor.isServer) {
   // on the client instead of copy/pasting the route path manually between the
   // client and the server.
   /**
-   * @operation export
+   * @operation exportJson
    * @tag Boards
    *
-   * @summary This route is used to export the board.
+   * @summary This route is used to export the board to a json file format.
    *
    * @description If user is already logged-in, pass loginToken as param
    * "authToken": '/api/boards/:boardId/export?authToken=:token'
@@ -46,199 +47,52 @@ if (Meteor.isServer) {
       JsonRoutes.sendResult(res, 403);
     }
   });
-}
-
-// exporter maybe is broken since Gridfs introduced, add fs and path
-
-export class Exporter {
-  constructor(boardId) {
-    this._boardId = boardId;
-  }
-
-  build() {
-    const fs = Npm.require('fs');
-    const os = Npm.require('os');
-    const path = Npm.require('path');
-
-    const byBoard = { boardId: this._boardId };
-    const byBoardNoLinked = {
-      boardId: this._boardId,
-      linkedId: { $in: ['', null] },
-    };
-    // we do not want to retrieve boardId in related elements
-    const noBoardId = {
-      fields: {
-        boardId: 0,
-      },
-    };
-    const result = {
-      _format: 'wekan-board-1.0.0',
-    };
-    _.extend(
-      result,
-      Boards.findOne(this._boardId, {
-        fields: {
-          stars: 0,
-        },
-      }),
-    );
-    result.lists = Lists.find(byBoard, noBoardId).fetch();
-    result.cards = Cards.find(byBoardNoLinked, noBoardId).fetch();
-    result.swimlanes = Swimlanes.find(byBoard, noBoardId).fetch();
-    result.customFields = CustomFields.find(
-      { boardIds: { $in: [this.boardId] } },
-      { fields: { boardId: 0 } },
-    ).fetch();
-    result.comments = CardComments.find(byBoard, noBoardId).fetch();
-    result.activities = Activities.find(byBoard, noBoardId).fetch();
-    result.rules = Rules.find(byBoard, noBoardId).fetch();
-    result.checklists = [];
-    result.checklistItems = [];
-    result.subtaskItems = [];
-    result.triggers = [];
-    result.actions = [];
-    result.cards.forEach(card => {
-      result.checklists.push(
-        ...Checklists.find({
-          cardId: card._id,
-        }).fetch(),
-      );
-      result.checklistItems.push(
-        ...ChecklistItems.find({
-          cardId: card._id,
-        }).fetch(),
-      );
-      result.subtaskItems.push(
-        ...Cards.find({
-          parentId: card._id,
-        }).fetch(),
-      );
-    });
-    result.rules.forEach(rule => {
-      result.triggers.push(
-        ...Triggers.find(
-          {
-            _id: rule.triggerId,
-          },
-          noBoardId,
-        ).fetch(),
-      );
-      result.actions.push(
-        ...Actions.find(
-          {
-            _id: rule.actionId,
-          },
-          noBoardId,
-        ).fetch(),
-      );
-    });
-
-    // [Old] for attachments we only export IDs and absolute url to original doc
-    // [New] Encode attachment to base64
-
-    const getBase64Data = function(doc, callback) {
-      let buffer = Buffer.allocUnsafe(0);
-      buffer.fill(0);
-
-      // callback has the form function (err, res) {}
-      const tmpFile = path.join(
-        os.tmpdir(),
-        `tmpexport${process.pid}${Math.random()}`,
-      );
-      const tmpWriteable = fs.createWriteStream(tmpFile);
-      const readStream = fs.createReadStream(doc.path);
-      readStream.on('data', function(chunk) {
-        buffer = Buffer.concat([buffer, chunk]);
-      });
-
-      readStream.on('error', function(err) {
-        callback(null, null);
-      });
-      readStream.on('end', function() {
-        // done
-        fs.unlink(tmpFile, () => {
-          //ignored
-        });
 
-        callback(null, buffer.toString('base64'));
+  /**
+   * @operation exportCSV/TSV
+   * @tag Boards
+   *
+   * @summary This route is used to export the board to a CSV or TSV file format.
+   *
+   * @description If user is already logged-in, pass loginToken as param
+   *
+   * See https://blog.kayla.com.au/server-side-route-authentication-in-meteor/
+   * for detailed explanations
+   *
+   * @param {string} boardId the ID of the board we are exporting
+   * @param {string} authToken the loginToken
+   * @param {string} delimiter delimiter to use while building export. Default is comma ','
+   */
+  Picker.route('/api/boards/:boardId/export/csv', function(params, req, res) {
+    const boardId = params.boardId;
+    let user = null;
+    const loginToken = params.query.authToken;
+    if (loginToken) {
+      const hashToken = Accounts._hashLoginToken(loginToken);
+      user = Meteor.users.findOne({
+        'services.resume.loginTokens.hashedToken': hashToken,
       });
-      readStream.pipe(tmpWriteable);
-    };
-    const getBase64DataSync = Meteor.wrapAsync(getBase64Data);
-    result.attachments = Attachments.find({ 'meta.boardId': byBoard.boardId })
-      .fetch()
-      .map(attachment => {
-        let filebase64 = null;
-        filebase64 = getBase64DataSync(attachment);
-
-        return {
-          _id: attachment._id,
-          cardId: attachment.meta.cardId,
-          //url: FlowRouter.url(attachment.url()),
-          file: filebase64,
-          name: attachment.name,
-          type: attachment.type,
-        };
+    } else if (!Meteor.settings.public.sandstorm) {
+      Authentication.checkUserId(req.userId);
+      user = Users.findOne({
+        _id: req.userId,
+        isAdmin: true,
       });
-
-    // we also have to export some user data - as the other elements only
-    // include id but we have to be careful:
-    // 1- only exports users that are linked somehow to that board
-    // 2- do not export any sensitive information
-    const users = {};
-    result.members.forEach(member => {
-      users[member.userId] = true;
-    });
-    result.lists.forEach(list => {
-      users[list.userId] = true;
-    });
-    result.cards.forEach(card => {
-      users[card.userId] = true;
-      if (card.members) {
-        card.members.forEach(memberId => {
-          users[memberId] = true;
-        });
-      }
-    });
-    result.comments.forEach(comment => {
-      users[comment.userId] = true;
-    });
-    result.activities.forEach(activity => {
-      users[activity.userId] = true;
-    });
-    result.checklists.forEach(checklist => {
-      users[checklist.userId] = true;
-    });
-    const byUserIds = {
-      _id: {
-        $in: Object.getOwnPropertyNames(users),
-      },
-    };
-    // we use whitelist to be sure we do not expose inadvertently
-    // some secret fields that gets added to User later.
-    const userFields = {
-      fields: {
-        _id: 1,
-        username: 1,
-        'profile.fullname': 1,
-        'profile.initials': 1,
-        'profile.avatarUrl': 1,
-      },
-    };
-    result.users = Users.find(byUserIds, userFields)
-      .fetch()
-      .map(user => {
-        // user avatar is stored as a relative url, we export absolute
-        if ((user.profile || {}).avatarUrl) {
-          user.profile.avatarUrl = FlowRouter.url(user.profile.avatarUrl);
-        }
-        return user;
+    }
+    const exporter = new Exporter(boardId);
+    if (exporter.canExport(user)) {
+      body = params.query.delimiter
+        ? exporter.buildCsv(params.query.delimiter)
+        : exporter.buildCsv();
+      res.writeHead(200, {
+        'Content-Length': body[0].length,
+        'Content-Type': params.query.delimiter ? 'text/csv' : 'text/tsv',
       });
-    return result;
-  }
-
-  canExport(user) {
-    const board = Boards.findOne(this._boardId);
-    return board && board.isVisibleBy(user);
-  }
+      res.write(body[0]);
+      res.end();
+    } else {
+      res.writeHead(403);
+      res.end('Permission Error');
+    }
+  });
 }

models/trelloCreator.js

@@ -369,7 +369,6 @@ export class TrelloCreator {
           // so we make it server only, and let UI catch up once it is done, forget about latency comp.
           const self = this;
           if (Meteor.isServer) {
-            // FIXME: Change to new model
             file.attachData(att.url, function(error) {
               file.boardId = boardId;
               file.cardId = cardId;

models/wekanCreator.js

@@ -415,7 +415,6 @@ export class WekanCreator {
           const self = this;
           if (Meteor.isServer) {
             if (att.url) {
-              // FIXME: Change to new file library
               file.attachData(att.url, function(error) {
                 file.boardId = boardId;
                 file.cardId = cardId;
@@ -441,7 +440,6 @@ export class WekanCreator {
                 }
               });
             } else if (att.file) {
-              // FIXME: Change to new file library
               file.attachData(
                 Buffer.from(att.file, 'base64'),
                 {

sandstorm-pkgdef.capnp

@@ -22,10 +22,10 @@ const pkgdef :Spk.PackageDefinition = (
     appTitle = (defaultText = "Wekan"),
     # The name of the app as it is displayed to the user.
 
-    appVersion = 404,
+    appVersion = 403,
     # Increment this for every release.
 
-    appMarketingVersion = (defaultText = "4.04.0~2020-05-24"),
+    appMarketingVersion = (defaultText = "4.03.0~2020-05-16"),
     # Human-readable presentation of the app version.
 
     minUpgradableAppVersion = 0,
@@ -261,7 +261,6 @@ const myCommand :Spk.Manifest.Command = (
     (key = "LDAP_ENABLE", value="false"),
     (key = "PASSWORD_LOGIN_ENABLED", value="true"),
     (key = "SANDSTORM", value="1"),
-    (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}"),
-    (key = "ATTACHMENTS_STORE_PATH", value = "/var/attachments/")
+    (key = "METEOR_SETTINGS", value = "{\"public\": {\"sandstorm\": true}}")
   ]
 );

server/migrations.js

@@ -80,7 +80,7 @@ Migrations.add('lowercase-board-permission', () => {
 Migrations.add('change-attachments-type-for-non-images', () => {
   const newTypeForNonImage = 'application/octet-stream';
   Attachments.find().forEach(file => {
-    if (!file.isImage) {
+    if (!file.isImage()) {
       Attachments.update(
         file._id,
         {
@@ -1044,46 +1044,3 @@ Migrations.add('add-sort-field-to-boards', () => {
     }
   });
 });
-
-import { MongoInternals } from 'meteor/mongo';
-
-Migrations.add('change-attachment-library', () => {
-	const fs = require('fs');
-	CFSAttachments.find().forEach(file => {
-    const bucket = new MongoInternals.NpmModule.GridFSBucket(MongoInternals.defaultRemoteCollectionDriver().mongo.db, {bucketName: 'cfs_gridfs.attachments'});
-    const gfsId = new MongoInternals.NpmModule.ObjectID(file.copies.attachments.key);
- 	  const reader = bucket.openDownloadStream(gfsId);
-    let store = Attachments.storagePath();
-    if (store.charAt(store.length - 1) === '/') {
-      store = store.substring(0, store.length - 1);
-    }
-		const path = `${store}/${file.name()}`;
-		const fd = fs.createWriteStream(path);
-		reader.pipe(fd);
-    reader.on('end', () => {
-      let opts = {
-        fileName: file.name(),
-        type: file.type(),
-        size: file.size(),
-        fileId: file._id,
-        meta: {
-          userId: file.userId,
-          boardId: file.boardId,
-          cardId: file.cardId
-        }
-      };
-      if (file.listId) {
-        opts.meta.listId = file.listId;
-      }
-      if (file.swimlaneId) {
-        opts.meta.swimlaneId = file.swimlaneId;
-      }
-      Attachments.addFile(path, opts, (err, fileRef) => {
-        if (err) {
-          console.log('error when migrating', file.name(), err);
-        }
-      });
-    });
-	});
-});
-

server/old-attachments-migration.js

@@ -1,212 +0,0 @@
-const localFSStore = process.env.ATTACHMENTS_STORE_PATH;
-const storeName = 'attachments';
-const defaultStoreOptions = {
-  beforeWrite: fileObj => {
-    if (!fileObj.isImage()) {
-      return {
-        type: 'application/octet-stream',
-      };
-    }
-    return {};
-  },
-};
-let store;
-if (localFSStore) {
-  // have to reinvent methods from FS.Store.GridFS and FS.Store.FileSystem
-  const fs = Npm.require('fs');
-  const path = Npm.require('path');
-  const mongodb = Npm.require('mongodb');
-  const Grid = Npm.require('gridfs-stream');
-  // calulate the absolute path here, because FS.Store.FileSystem didn't expose the aboslutepath or FS.Store didn't expose api calls :(
-  let pathname = localFSStore;
-  /*eslint camelcase: ["error", {allow: ["__meteor_bootstrap__"]}] */
-
-  if (!pathname && __meteor_bootstrap__ && __meteor_bootstrap__.serverDir) {
-    pathname = path.join(
-      __meteor_bootstrap__.serverDir,
-      `../../../cfs/files/${storeName}`,
-    );
-  }
-
-  if (!pathname)
-    throw new Error('FS.Store.FileSystem unable to determine path');
-
-  // Check if we have '~/foo/bar'
-  if (pathname.split(path.sep)[0] === '~') {
-    const homepath =
-      process.env.HOME || process.env.HOMEPATH || process.env.USERPROFILE;
-    if (homepath) {
-      pathname = pathname.replace('~', homepath);
-    } else {
-      throw new Error('FS.Store.FileSystem unable to resolve "~" in path');
-    }
-  }
-
-  // Set absolute path
-  const absolutePath = path.resolve(pathname);
-
-  const _FStore = new FS.Store.FileSystem(storeName, {
-    path: localFSStore,
-    ...defaultStoreOptions,
-  });
-  const GStore = {
-    fileKey(fileObj) {
-      const key = {
-        _id: null,
-        filename: null,
-      };
-
-      // If we're passed a fileObj, we retrieve the _id and filename from it.
-      if (fileObj) {
-        const info = fileObj._getInfo(storeName, {
-          updateFileRecordFirst: false,
-        });
-        key._id = info.key || null;
-        key.filename =
-          info.name ||
-          fileObj.name({ updateFileRecordFirst: false }) ||
-          `${fileObj.collectionName}-${fileObj._id}`;
-      }
-
-      // If key._id is null at this point, createWriteStream will let GridFS generate a new ID
-      return key;
-    },
-    db: undefined,
-    mongoOptions: { useNewUrlParser: true },
-    mongoUrl: process.env.MONGO_URL,
-    init() {
-      this._init(err => {
-        this.inited = !err;
-      });
-    },
-    _init(callback) {
-      const self = this;
-      mongodb.MongoClient.connect(self.mongoUrl, self.mongoOptions, function(
-        err,
-        db,
-      ) {
-        if (err) {
-          return callback(err);
-        }
-        self.db = db;
-        return callback(null);
-      });
-      return;
-    },
-    createReadStream(fileKey, options) {
-      const self = this;
-      if (!self.inited) {
-        self.init();
-        return undefined;
-      }
-      options = options || {};
-
-      // Init GridFS
-      const gfs = new Grid(self.db, mongodb);
-
-      // Set the default streamning settings
-      const settings = {
-        _id: new mongodb.ObjectID(fileKey._id),
-        root: `cfs_gridfs.${storeName}`,
-      };
-
-      // Check if this should be a partial read
-      if (
-        typeof options.start !== 'undefined' &&
-        typeof options.end !== 'undefined'
-      ) {
-        // Add partial info
-        settings.range = {
-          startPos: options.start,
-          endPos: options.end,
-        };
-      }
-      return gfs.createReadStream(settings);
-    },
-  };
-  GStore.init();
-  const CRS = 'createReadStream';
-  const _CRS = `_${CRS}`;
-  const FStore = _FStore._transform;
-  FStore[_CRS] = FStore[CRS].bind(FStore);
-  FStore[CRS] = function(fileObj, options) {
-    let stream;
-    try {
-      const localFile = path.join(
-        absolutePath,
-        FStore.storage.fileKey(fileObj),
-      );
-      const state = fs.statSync(localFile);
-      if (state) {
-        stream = FStore[_CRS](fileObj, options);
-      }
-    } catch (e) {
-      // file is not there, try GridFS ?
-      stream = undefined;
-    }
-    if (stream) return stream;
-    else {
-      try {
-        const stream = GStore[CRS](GStore.fileKey(fileObj), options);
-        return stream;
-      } catch (e) {
-        return undefined;
-      }
-    }
-  }.bind(FStore);
-  store = _FStore;
-} else {
-  store = new FS.Store.GridFS(localFSStore ? `G${storeName}` : storeName, {
-    // XXX Add a new store for cover thumbnails so we don't load big images in
-    // the general board view
-    // If the uploaded document is not an image we need to enforce browser
-    // download instead of execution. This is particularly important for HTML
-    // files that the browser will just execute if we don't serve them with the
-    // appropriate `application/octet-stream` MIME header which can lead to user
-    // data leaks. I imagine other formats (like PDF) can also be attack vectors.
-    // See https://github.com/wekan/wekan/issues/99
-    // XXX Should we use `beforeWrite` option of CollectionFS instead of
-    // collection-hooks?
-    // We should use `beforeWrite`.
-    ...defaultStoreOptions,
-  });
-}
-CFSAttachments = new FS.Collection('attachments', {
-  stores: [store],
-});
-
-if (Meteor.isServer) {
-  Meteor.startup(() => {
-    CFSAttachments.files._ensureIndex({ cardId: 1 });
-  });
-
-  CFSAttachments.allow({
-    insert(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    update(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    remove(userId, doc) {
-      return allowIsBoardMember(userId, Boards.findOne(doc.boardId));
-    },
-    // We authorize the attachment download either:
-    // - if the board is public, everyone (even unconnected) can download it
-    // - if the board is private, only board members can download it
-    download(userId, doc) {
-      if (Meteor.isServer) {
-        return true;
-      }
-      const board = Boards.findOne(doc.boardId);
-      if (board.isPublic()) {
-        return true;
-      } else {
-        return board.hasMember(userId);
-      }
-    },
-
-    fetch: ['boardId'],
-  });
-}
-
-export default CFSAttachments;

server/publications/boards.js

@@ -131,7 +131,7 @@ Meteor.publishRelations('board', function(boardId, isArchived) {
       // Gather queries and send in bulk
       const cardComments = this.join(CardComments);
       cardComments.selector = _ids => ({ cardId: _ids });
-      const attachments = this.join(Attachments.collection);
+      const attachments = this.join(Attachments);
       attachments.selector = _ids => ({ cardId: _ids });
       const checklists = this.join(Checklists);
       checklists.selector = _ids => ({ cardId: _ids });

snap-src/bin/config

@@ -93,9 +93,11 @@ DEFAULT_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="15"
 KEY_ACCOUNTS_LOCKOUT_UNKNOWN_USERS_FAILURE_WINDOW="accounts-lockout-unknown-users-failure-window"
 
 DESCRIPTION_ATTACHMENTS_STORE_PATH="Allow wekan ower to specify where uploaded files to store on the server instead of the mongodb"
-DEFAULT_ATTACHMENTS_STORE_PATH="/var/snap/wekan/common/uploads/"
+DEFAULT_ATTACHMENTS_STORE_PATH=""
 KEY_ATTACHMENTS_STORE_PATH="attachments-store-path"
 
+# Example, not in use: /var/snap/wekan/common/uploads/
+
 DESCRIPTION_MAX_IMAGE_PIXEL="Max image pixel: Allow to shrink attached/pasted image https://github.com/wekan/wekan/pull/2544"
 DEFAULT_MAX_IMAGE_PIXEL=""
 KEY_MAX_IMAGE_PIXEL="max-image-pixel"

snap-src/bin/mongodb-control

@@ -24,11 +24,11 @@ if test -f "$SNAP_COMMON/mongodb.log"; then
    rm -f "$SNAP_COMMON/mongodb.log"
 fi
 
-# If uploads directory does not exist, create it.
+# Not in use. If uploads directory does not exist, create it.
 # Wekan will store attachments there.
-if [ ! -d "$SNAP_COMMON/uploads" ]; then
-   mkdir "$SNAP_COMMON/uploads"
-fi
+#if [ ! -d "$SNAP_COMMON/uploads" ]; then
+#   mkdir "$SNAP_COMMON/uploads"
+#fi
 
 # Alternative: When starting MongoDB, and using logfile, truncate log to last 1000 lines of text.
 # 1) If file exists: