Adding keywords script

Author: runemadsen

scripts/createKeywords.js

@@ -40,8 +40,10 @@ const path = require('path');
 // OTHER changes
 // draw has been changed to draw_ on the righthand side
 // Gobal functions currently have _ at end, which they didn't in the original file: keyPressed, keyReleased, keyTyped, mouseClicked, mouseDragged, mouseMoved, mousePressed, mouseReleased, mouseWheel, settings, setup
-// pixelHeight was listed as a function with _ at the right end. I removed it.
+// pixelHeight was there twice. I removed it from the generated code
 // PVector was listed twice as FUNCTION1 in base and KEYWORD5 in generated output. I keps the latter.
+// PShader.set was `PShader	FUNCTION2	PShader_set_`. Changed to `set	FUNCTION2	PShader_set_`
+// PFont.list was listed as FUNCTION1. I changed it to be FUNCTION2 like the other class methods.
 
 const folder = path.join(
   __dirname,
@@ -59,81 +61,99 @@ const folder = path.join(
 **/
 const ignore = ['width', 'height', 'x', 'y', 'z', 'Object'];
 
+/**
+  A list of global functions that need to have special handling
+**/
+const globalFuncs = [
+  'keyPressed()',
+  'keyReleased()',
+  'keyTyped()',
+  'mouseClicked()',
+  'mouseDragged()',
+  'mouseMoved()',
+  'mousePressed()',
+  'mouseReleased()',
+  'mouseWheel()',
+  'settings()',
+  'setup()',
+  'draw()'
+];
+
 /**
   This script creates the keywords.txt file that is used to do syntax highligting
   in the Processing IDE.
 **/
 const createKeywords = () => {
-  // TEMPORARY: To be used for diffing
-  const currentKeywords = parseKeywordsFile('current_keywords.txt');
-  const absIndex = currentKeywords.findIndex((k) => k[0] === 'abs');
-  const needToGenerate = currentKeywords.slice(absIndex);
-
   // Load the base keywords
-  const baseKeywords = parseKeywordsFile('keywords_base.txt');
+  const [baseKeywords, baseContents] = parseKeywordsFile('keywords_base.txt');
 
   // Load all json files
   const entries = fs.readdirSync(folder).map((file) => {
-    const entry = require(path.join(folder, file));
-    return Object.assign(
+    let entry = require(path.join(folder, file));
+    entry = Object.assign(
       {
         filename: file,
-        basename: path.basename(file, '.json'),
-        clean: cleanName(entry)
+        basename: path.basename(file, '.json')
       },
       entry
     );
+    entry.leftName = leftName(entry);
+    entry.rightName = rightName(entry);
+    entry.token = getToken(entry);
+    return entry;
   });
 
   // Sort by listing in original keywords.txt
   entries.sort(sortByClassAndGlobal);
 
   // Run through and add if not already in base
-  const newKeywords = [];
+  const keywords = [];
   for (let i = 0; i < entries.length; i++) {
     const entry = entries[i];
-    if (entry.name === 'PVector') {
-      console.log('here', shouldInclude(baseKeywords, entry));
-    }
     if (shouldInclude(baseKeywords, entry)) {
-      newKeywords.push([entry.clean, 'CATEGORY', entry.basename]);
+      keywords.push([entry.leftName, entry.token, entry.rightName]);
     }
   }
 
-  // Log to compare objects
-  for (let i = 0; i < needToGenerate.length; i++) {
-    const a = needToGenerate[i][2];
-    const b = newKeywords[i] ? newKeywords[i][2] : null;
-    console.log(a == b, `| ${a} | ${b} |`);
-  }
+  const pad = (str, size) => (str ? str.padEnd(size, ' ') : str);
+
+  // COMPARE TO OLD VERSION
+  /*
+  const [currentKeywords] = parseKeywordsFile('keywords_old.txt');
+  const absIndex = currentKeywords.findIndex((k) => k[0] === 'abs');
+  const needToGenerate = currentKeywords.slice(absIndex);
+  console.log('Old', needToGenerate.length, 'New', keywords.length);
 
-  console.log(
-    'Old length',
-    needToGenerate.length,
-    'New Length',
-    newKeywords.length
-  );
-
-  // output rest of stuff from newKeywords
-  if (newKeywords.length > needToGenerate.length) {
-    for (let i = needToGenerate.length; i < newKeywords.length; i++) {
-      console.log('EXTRA', needToGenerate[i]);
+  for (let i = 0; i < needToGenerate.length; i++) {
+    const a = needToGenerate[i];
+    const b = keywords[i];
+    const match = a[0] === b[0] && a[1] === b[1] && a[2] === b[2];
+
+    if (!match) {
+      console.log(
+        pad(match.toString(), 7),
+        pad(a[0], 25),
+        pad(b[0], 25),
+        pad(a[1], 10),
+        pad(b[1], 10),
+        pad(a[2], 25),
+        pad(b[2], 25)
+      );
     }
   }
-};
+  */
 
-const sortByClassAndGlobal = (a, b) => {
-  // If a is not a class, but b is, a comes first
-  // if (!a.classanchor && b.classanchor) {
-  //   return -1;
-  // }
+  // Save to keywords.txt file
+  const combined = baseContents + '\n\n' + generateKeywordsFile(keywords);
 
-  // If a is a class, but b is not, a comes last
-  // else if (a.classanchor && !b.classanchor) {
-  //   return 1;
-  // }
+  fs.writeFileSync(path.join(__dirname, 'keywords.txt'), combined);
+  console.log('keywords.txt file written to scripts folder');
+};
 
-  // Otherwise, compare apples to apples
+/**
+  Sorts the entries based on the basename
+**/
+const sortByClassAndGlobal = (a, b) => {
   const strippedA = a.basename.replace(/_$/, '').toLowerCase();
   const strippedB = b.basename.replace(/_$/, '').toLowerCase();
 
@@ -149,7 +169,7 @@ const sortByClassAndGlobal = (a, b) => {
 };
 
 /**
-  Looks to see if the entry is in the keywords base.
+  Looks to see if the entry is in the base keywords.
 **/
 const shouldInclude = (keywords, entry) => {
   // Don't include ignored names
@@ -159,7 +179,7 @@ const shouldInclude = (keywords, entry) => {
 
   // See if entry is in the keywords base
   for (let i = 0; i < keywords.length; i++) {
-    if (keywords[i][0] == entry.clean && keywords[i][2] == entry.basename) {
+    if (keywords[i][0] == entry.leftName && keywords[i][2] == entry.basename) {
       return false;
     }
   }
@@ -168,9 +188,12 @@ const shouldInclude = (keywords, entry) => {
   return true;
 };
 
+/**
+  Parse a keywords file into an array of arrays
+**/
 const parseKeywordsFile = (filename) => {
-  const txt = fs.readFileSync(path.join(__dirname, filename)).toString();
-  const lines = txt.split('\n');
+  const content = fs.readFileSync(path.join(__dirname, filename)).toString();
+  const lines = content.split('\n');
   const keywords = [];
 
   for (let i = 0; i < lines.length; i++) {
@@ -180,7 +203,15 @@ const parseKeywordsFile = (filename) => {
     }
   }
 
-  return keywords;
+  return [keywords, content];
+};
+
+const generateKeywordsFile = (keywords) => {
+  let str = '';
+  for (let i = 0; i < keywords.length; i++) {
+    str += `${keywords[i][0]}\t${keywords[i][1]}\t${keywords[i][2]}\n`;
+  }
+  return str;
 };
 
 /**
@@ -195,36 +226,61 @@ const specialCases = {
 /**
   Makes the clean version of the name for the left-hand side of the file
 **/
-const cleanName = (entry) => {
-  // Constants, classes and primitives are just using the name
-  if (
-    entry.type === 'class' ||
-    entry.category === 'constants' ||
-    (entry.category === 'Data' && entry.subcategory === 'Primitive')
-  ) {
-    return entry.name;
-  }
-
+const operatorRegex = /\s\([a-zA-Z\s]+\)$/;
+const leftName = (entry) => {
   // Special handling
   if (specialCases[entry.name]) {
     return specialCases[entry.name];
   }
 
   // Operators are renamed: += (add assign) => +=
-  if (
-    entry.subcategory === 'Operators' ||
-    entry.subcategory == 'Bitwise Operators' ||
-    entry.subcategory == 'Relational Operators' ||
-    entry.subcategory == 'Logical Operators' ||
-    (entry.category === 'structure' && entry.subcategory === '')
-  ) {
-    return entry.name.replace(/\([a-zA-Z\s]+\)$/, '').trim();
+  if (entry.name.match(operatorRegex)) {
+    return entry.name.replace(operatorRegex, '');
   }
 
   // Functions and class methods are renamed: float() => float
   if (entry.type === 'function' || entry.type === 'method') {
     return entry.name.replace(/\(\)$/, '');
   }
+
+  // Variables are renamed: pixels[] => pixels
+  if (entry.type === 'field' || entry.type === 'other') {
+    return entry.name.replace(/\[\]$/, '');
+  }
+
+  // Just use name (Constants, classes, primitives, etc)
+  return entry.name;
+};
+
+/**
+  Make a name suitable for the right hand side
+**/
+const rightName = (entry) => {
+  // global functions appear as just the name in the right
+  if (globalFuncs.includes(entry.name)) {
+    return entry.name.replace(/\(\)$/, '');
+  }
+
+  // Everything else uses the basename (fill_, PVector_add_, etc)
+  return entry.basename;
+};
+
+/**
+  Gets the syntax category. Based on the old pearl script.
+**/
+const getToken = (entry) => {
+  if (globalFuncs.includes(entry.name)) {
+    return 'FUNCTION4';
+  } else if (entry.type === 'class') {
+    return 'KEYWORD5';
+  } else if (entry.type === 'method') {
+    return 'FUNCTION2';
+  } else if (entry.type === 'other') {
+    return 'KEYWORD4';
+  } else if (entry.type === 'field') {
+    return 'KEYWORD2';
+  }
+  return 'FUNCTION1';
 };
 
 createKeywords();