diff --git a/.gitignore b/.gitignore
index 42b1bde94..2ae0f98c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
/node_modules
/built
/uploads
+/data
npm-debug.log
*.pem
run.bat
diff --git a/locales/en.yml b/locales/en.yml
index d40896212..3b87ea758 100644
--- a/locales/en.yml
+++ b/locales/en.yml
@@ -22,6 +22,14 @@ common:
confused: "Confused"
pudding: "Pudding"
+ post_categories:
+ music: "Music"
+ game: "Video Game"
+ anime: "Anime"
+ it: "IT"
+ gadgets: "Gadgets"
+ photography: "Photography"
+
input-message-here: "Enter message here"
send: "Send"
delete: "Delete"
@@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "Pin"
pinned: "Pinned"
+ select: "Select category"
+ categorize: "Accept"
+ categorized: "Category reported. Thank you!"
mk-reaction-picker:
choose-reaction: "Pick your reaction"
@@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter integration"
signin-history: "Sign in history"
api: "API"
+ link: "MisskeyLink"
settings: "Settings"
signout: "Sign out"
diff --git a/locales/ja.yml b/locales/ja.yml
index b8e5cff41..13d451b6d 100644
--- a/locales/ja.yml
+++ b/locales/ja.yml
@@ -22,6 +22,14 @@ common:
confused: "こまこまのこまり"
pudding: "Pudding"
+ post_categories:
+ music: "音楽"
+ game: "ゲーム"
+ anime: "アニメ"
+ it: "IT"
+ gadgets: "ガジェット"
+ photography: "写真"
+
input-message-here: "ここにメッセージを入力"
send: "送信"
delete: "削除"
@@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "ピン留め"
pinned: "ピン留めしました"
+ select: "カテゴリを選択"
+ categorize: "決定"
+ categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。"
mk-reaction-picker:
choose-reaction: "リアクションを選択"
@@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter連携"
signin-history: "ログイン履歴"
api: "API"
+ link: "Misskeyリンク"
settings: "設定"
signout: "サインアウト"
diff --git a/package.json b/package.json
index a2896f4c7..31cf7a02c 100644
--- a/package.json
+++ b/package.json
@@ -64,6 +64,7 @@
"@types/webpack": "3.0.10",
"@types/webpack-stream": "3.2.7",
"@types/websocket": "0.0.34",
+ "@types/msgpack-lite": "^0.1.5",
"chai": "4.1.2",
"chai-http": "3.0.0",
"css-loader": "0.28.7",
@@ -120,10 +121,12 @@
"is-root": "1.0.0",
"is-url": "1.2.2",
"js-yaml": "3.9.1",
+ "mecab-async": "^0.1.0",
"mongodb": "2.2.31",
"monk": "6.0.3",
"morgan": "1.8.2",
"ms": "2.0.0",
+ "msgpack-lite": "^0.1.26",
"multer": "1.3.0",
"nprogress": "0.2.0",
"os-utils": "0.0.14",
diff --git a/src/api/endpoints.ts b/src/api/endpoints.ts
index e5be68c09..97b98895b 100644
--- a/src/api/endpoints.ts
+++ b/src/api/endpoints.ts
@@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [
name: 'posts/trend',
withCredential: true
},
+ {
+ name: 'posts/categorize',
+ withCredential: true
+ },
{
name: 'posts/reactions',
withCredential: true
diff --git a/src/api/endpoints/posts/categorize.ts b/src/api/endpoints/posts/categorize.ts
new file mode 100644
index 000000000..3530ba6bc
--- /dev/null
+++ b/src/api/endpoints/posts/categorize.ts
@@ -0,0 +1,52 @@
+/**
+ * Module dependencies
+ */
+import $ from 'cafy';
+import Post from '../../models/post';
+
+/**
+ * Categorize a post
+ *
+ * @param {any} params
+ * @param {any} user
+ * @return {Promise<any>}
+ */
+module.exports = (params, user) => new Promise(async (res, rej) => {
+ if (!user.is_pro) {
+ return rej('This endpoint is available only from a Pro account');
+ }
+
+ // Get 'post_id' parameter
+ const [postId, postIdErr] = $(params.post_id).id().$;
+ if (postIdErr) return rej('invalid post_id param');
+
+ // Get categorizee
+ const post = await Post.findOne({
+ _id: postId
+ });
+
+ if (post === null) {
+ return rej('post not found');
+ }
+
+ if (post.is_category_verified) {
+ return rej('This post already has the verified category');
+ }
+
+ // Get 'category' parameter
+ const [category, categoryErr] = $(params.category).string().or([
+ 'music', 'game', 'anime', 'it', 'gadgets', 'photography'
+ ]).$;
+ if (categoryErr) return rej('invalid category param');
+
+ // Set category
+ Post.update({ _id: post._id }, {
+ $set: {
+ category: category,
+ is_category_verified: true
+ }
+ });
+
+ // Send response
+ res();
+});
diff --git a/src/config.ts b/src/config.ts
index 8f4ada5af..f333a1f5a 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -68,6 +68,9 @@ type Source = {
hook_secret: string;
username: string;
};
+ categorizer?: {
+ mecab_command?: string;
+ };
};
/**
diff --git a/src/tools/ai/naive-bayes.js b/src/tools/ai/naive-bayes.js
new file mode 100644
index 000000000..78f07153c
--- /dev/null
+++ b/src/tools/ai/naive-bayes.js
@@ -0,0 +1,302 @@
+// Original source code: https://github.com/ttezel/bayes/blob/master/lib/naive_bayes.js (commit: 2c20d3066e4fc786400aaedcf3e42987e52abe3c)
+// CUSTOMIZED BY SYUILO
+
+/*
+ Expose our naive-bayes generator function
+*/
+module.exports = function (options) {
+ return new Naivebayes(options)
+}
+
+// keys we use to serialize a classifier's state
+var STATE_KEYS = module.exports.STATE_KEYS = [
+ 'categories', 'docCount', 'totalDocuments', 'vocabulary', 'vocabularySize',
+ 'wordCount', 'wordFrequencyCount', 'options'
+];
+
+/**
+ * Initializes a NaiveBayes instance from a JSON state representation.
+ * Use this with classifier.toJson().
+ *
+ * @param {String} jsonStr state representation obtained by classifier.toJson()
+ * @return {NaiveBayes} Classifier
+ */
+module.exports.fromJson = function (jsonStr) {
+ var parsed;
+ try {
+ parsed = JSON.parse(jsonStr)
+ } catch (e) {
+ throw new Error('Naivebayes.fromJson expects a valid JSON string.')
+ }
+ // init a new classifier
+ var classifier = new Naivebayes(parsed.options)
+
+ // override the classifier's state
+ STATE_KEYS.forEach(function (k) {
+ if (!parsed[k]) {
+ throw new Error('Naivebayes.fromJson: JSON string is missing an expected property: `'+k+'`.')
+ }
+ classifier[k] = parsed[k]
+ })
+
+ return classifier
+}
+
+/**
+ * Given an input string, tokenize it into an array of word tokens.
+ * This is the default tokenization function used if user does not provide one in `options`.
+ *
+ * @param {String} text
+ * @return {Array}
+ */
+var defaultTokenizer = function (text) {
+ //remove punctuation from text - remove anything that isn't a word char or a space
+ var rgxPunctuation = /[^(a-zA-ZA-Яa-я0-9_)+\s]/g
+
+ var sanitized = text.replace(rgxPunctuation, ' ')
+
+ return sanitized.split(/\s+/)
+}
+
+/**
+ * Naive-Bayes Classifier
+ *
+ * This is a naive-bayes classifier that uses Laplace Smoothing.
+ *
+ * Takes an (optional) options object containing:
+ * - `tokenizer` => custom tokenization function
+ *
+ */
+function Naivebayes (options) {
+ // set options object
+ this.options = {}
+ if (typeof options !== 'undefined') {
+ if (!options || typeof options !== 'object' || Array.isArray(options)) {
+ throw TypeError('NaiveBayes got invalid `options`: `' + options + '`. Pass in an object.')
+ }
+ this.options = options
+ }
+
+ this.tokenizer = this.options.tokenizer || defaultTokenizer
+
+ //initialize our vocabulary and its size
+ this.vocabulary = {}
+ this.vocabularySize = 0
+
+ //number of documents we have learned from
+ this.totalDocuments = 0
+
+ //document frequency table for each of our categories
+ //=> for each category, how often were documents mapped to it
+ this.docCount = {}
+
+ //for each category, how many words total were mapped to it
+ this.wordCount = {}
+
+ //word frequency table for each category
+ //=> for each category, how frequent was a given word mapped to it
+ this.wordFrequencyCount = {}
+
+ //hashmap of our category names
+ this.categories = {}
+}
+
+/**
+ * Initialize each of our data structure entries for this new category
+ *
+ * @param {String} categoryName
+ */
+Naivebayes.prototype.initializeCategory = function (categoryName) {
+ if (!this.categories[categoryName]) {
+ this.docCount[categoryName] = 0
+ this.wordCount[categoryName] = 0
+ this.wordFrequencyCount[categoryName] = {}
+ this.categories[categoryName] = true
+ }
+ return this
+}
+
+/**
+ * train our naive-bayes classifier by telling it what `category`
+ * the `text` corresponds to.
+ *
+ * @param {String} text
+ * @param {String} class
+ */
+Naivebayes.prototype.learn = function (text, category) {
+ var self = this
+
+ //initialize category data structures if we've never seen this category
+ self.initializeCategory(category)
+
+ //update our count of how many documents mapped to this category
+ self.docCount[category]++
+
+ //update the total number of documents we have learned from
+ self.totalDocuments++
+
+ //normalize the text into a word array
+ var tokens = self.tokenizer(text)
+
+ //get a frequency count for each token in the text
+ var frequencyTable = self.frequencyTable(tokens)
+
+ /*
+ Update our vocabulary and our word frequency count for this category
+ */
+
+ Object
+ .keys(frequencyTable)
+ .forEach(function (token) {
+ //add this word to our vocabulary if not already existing
+ if (!self.vocabulary[token]) {
+ self.vocabulary[token] = true
+ self.vocabularySize++
+ }
+
+ var frequencyInText = frequencyTable[token]
+
+ //update the frequency information for this word in this category
+ if (!self.wordFrequencyCount[category][token])
+ self.wordFrequencyCount[category][token] = frequencyInText
+ else
+ self.wordFrequencyCount[category][token] += frequencyInText
+
+ //update the count of all words we have seen mapped to this category
+ self.wordCount[category] += frequencyInText
+ })
+
+ return self
+}
+
+/**
+ * Determine what category `text` belongs to.
+ *
+ * @param {String} text
+ * @return {String} category
+ */
+Naivebayes.prototype.categorize = function (text) {
+ var self = this
+ , maxProbability = -Infinity
+ , chosenCategory = null
+
+ var tokens = self.tokenizer(text)
+ var frequencyTable = self.frequencyTable(tokens)
+
+ //iterate thru our categories to find the one with max probability for this text
+ Object
+ .keys(self.categories)
+ .forEach(function (category) {
+
+ //start by calculating the overall probability of this category
+ //=> out of all documents we've ever looked at, how many were
+ // mapped to this category
+ var categoryProbability = self.docCount[category] / self.totalDocuments
+
+ //take the log to avoid underflow
+ var logProbability = Math.log(categoryProbability)
+
+ //now determine P( w | c ) for each word `w` in the text
+ Object
+ .keys(frequencyTable)
+ .forEach(function (token) {
+ var frequencyInText = frequencyTable[token]
+ var tokenProbability = self.tokenProbability(token, category)
+
+ // console.log('token: %s category: `%s` tokenProbability: %d', token, category, tokenProbability)
+
+ //determine the log of the P( w | c ) for this word
+ logProbability += frequencyInText * Math.log(tokenProbability)
+ })
+
+ if (logProbability > maxProbability) {
+ maxProbability = logProbability
+ chosenCategory = category
+ }
+ })
+
+ return chosenCategory
+}
+
+/**
+ * Calculate probability that a `token` belongs to a `category`
+ *
+ * @param {String} token
+ * @param {String} category
+ * @return {Number} probability
+ */
+Naivebayes.prototype.tokenProbability = function (token, category) {
+ //how many times this word has occurred in documents mapped to this category
+ var wordFrequencyCount = this.wordFrequencyCount[category][token] || 0
+
+ //what is the count of all words that have ever been mapped to this category
+ var wordCount = this.wordCount[category]
+
+ //use laplace Add-1 Smoothing equation
+ return ( wordFrequencyCount + 1 ) / ( wordCount + this.vocabularySize )
+}
+
+/**
+ * Build a frequency hashmap where
+ * - the keys are the entries in `tokens`
+ * - the values are the frequency of each entry in `tokens`
+ *
+ * @param {Array} tokens Normalized word array
+ * @return {Object}
+ */
+Naivebayes.prototype.frequencyTable = function (tokens) {
+ var frequencyTable = Object.create(null)
+
+ tokens.forEach(function (token) {
+ if (!frequencyTable[token])
+ frequencyTable[token] = 1
+ else
+ frequencyTable[token]++
+ })
+
+ return frequencyTable
+}
+
+/**
+ * Dump the classifier's state as a JSON string.
+ * @return {String} Representation of the classifier.
+ */
+Naivebayes.prototype.toJson = function () {
+ var state = {}
+ var self = this
+ STATE_KEYS.forEach(function (k) {
+ state[k] = self[k]
+ })
+
+ var jsonStr = JSON.stringify(state)
+
+ return jsonStr
+}
+
+// (original method)
+Naivebayes.prototype.export = function () {
+ var state = {}
+ var self = this
+ STATE_KEYS.forEach(function (k) {
+ state[k] = self[k]
+ })
+
+ return state
+}
+
+module.exports.import = function (data) {
+ var parsed = data
+
+ // init a new classifier
+ var classifier = new Naivebayes()
+
+ // override the classifier's state
+ STATE_KEYS.forEach(function (k) {
+ if (!parsed[k]) {
+ throw new Error('Naivebayes.import: data is missing an expected property: `'+k+'`.')
+ }
+ classifier[k] = parsed[k]
+ })
+
+ return classifier
+}
diff --git a/src/tools/ai/predict-all-post-category.ts b/src/tools/ai/predict-all-post-category.ts
new file mode 100644
index 000000000..87e198b39
--- /dev/null
+++ b/src/tools/ai/predict-all-post-category.ts
@@ -0,0 +1,57 @@
+const bayes = require('./naive-bayes.js');
+const MeCab = require('mecab-async');
+
+import Post from '../../api/models/post';
+import config from '../../conf';
+
+const classifier = bayes({
+ tokenizer: this.tokenizer
+});
+
+const mecab = new MeCab();
+if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
+
+// 訓練データ取得
+Post.find({
+ is_category_verified: true
+}, {
+ fields: {
+ _id: false,
+ text: true,
+ category: true
+ }
+}).then(verifiedPosts => {
+ // 学習
+ verifiedPosts.forEach(post => {
+ classifier.learn(post.text, post.category);
+ });
+
+ // 全ての(人間によって証明されていない)投稿を取得
+ Post.find({
+ text: {
+ $exists: true
+ },
+ is_category_verified: {
+ $ne: true
+ }
+ }, {
+ sort: {
+ _id: -1
+ },
+ fields: {
+ _id: true,
+ text: true
+ }
+ }).then(posts => {
+ posts.forEach(post => {
+ console.log(`predicting... ${post._id}`);
+ const category = classifier.categorize(post.text);
+
+ Post.update({ _id: post._id }, {
+ $set: {
+ category: category
+ }
+ });
+ });
+ });
+});
diff --git a/src/tools/ai/predict-user-interst.ts b/src/tools/ai/predict-user-interst.ts
new file mode 100644
index 000000000..99bdfa420
--- /dev/null
+++ b/src/tools/ai/predict-user-interst.ts
@@ -0,0 +1,45 @@
+import Post from '../../api/models/post';
+import User from '../../api/models/user';
+
+export async function predictOne(id) {
+ console.log(`predict interest of ${id} ...`);
+
+ // TODO: repostなども含める
+ const recentPosts = await Post.find({
+ user_id: id,
+ category: {
+ $exists: true
+ }
+ }, {
+ sort: {
+ _id: -1
+ },
+ limit: 1000,
+ fields: {
+ _id: false,
+ category: true
+ }
+ });
+
+ const categories = {};
+
+ recentPosts.forEach(post => {
+ if (categories[post.category]) {
+ categories[post.category]++;
+ } else {
+ categories[post.category] = 1;
+ }
+ });
+}
+
+export async function predictAll() {
+ const allUsers = await User.find({}, {
+ fields: {
+ _id: true
+ }
+ });
+
+ allUsers.forEach(user => {
+ predictOne(user._id);
+ });
+}
diff --git a/src/web/app/common/tags/post-menu.tag b/src/web/app/common/tags/post-menu.tag
index 33895212b..be4468a21 100644
--- a/src/web/app/common/tags/post-menu.tag
+++ b/src/web/app/common/tags/post-menu.tag
@@ -2,6 +2,18 @@
<div class="backdrop" ref="backdrop" onclick={ close }></div>
<div class="popover { compact: opts.compact }" ref="popover">
<button if={ post.user_id === I.id } onclick={ pin }>%i18n:common.tags.mk-post-menu.pin%</button>
+ <div if={ I.is_pro && !post.is_category_verified }>
+ <select ref="categorySelect">
+ <option value="">%i18n:common.tags.mk-post-menu.select%</option>
+ <option value="music">%i18n:common.post_categories.music%</option>
+ <option value="game">%i18n:common.post_categories.game%</option>
+ <option value="anime">%i18n:common.post_categories.anime%</option>
+ <option value="it">%i18n:common.post_categories.it%</option>
+ <option value="gadgets">%i18n:common.post_categories.gadgets%</option>
+ <option value="photography">%i18n:common.post_categories.photography%</option>
+ </select>
+ <button onclick={ categorize }>%i18n:common.tags.mk-post-menu.categorize%</button>
+ </div>
</div>
<style>
$border-color = rgba(27, 31, 35, 0.15)
@@ -111,6 +123,17 @@
});
};
+ this.categorize = () => {
+ const category = this.refs.categorySelect.options[this.refs.categorySelect.selectedIndex].value;
+ this.api('posts/categorize', {
+ post_id: this.post.id,
+ category: category
+ }).then(() => {
+ if (this.opts.cb) this.opts.cb('categorized', '%i18n:common.tags.mk-post-menu.categorized%');
+ this.unmount();
+ });
+ };
+
this.close = () => {
this.refs.backdrop.style.pointerEvents = 'none';
anime({