CareerCup

Can you define "tokenized file"

Can you define what the tokenized file is ?

This is a very unusual Facebook question as normally is not technology specific but more specific to a problem.

Searching the internet I found how a class and function would look like.

<?php 
class Foo { 
    public $aMemberVar = 'aMemberVar Member Variable'; 
    public $aFuncName = 'aMemberFunc'; 
    
    
    function aMemberFunc() { 
        print 'Inside `aMemberFunc()`'; 
    } 
} 

$foo = new Foo; 
?>

I can parse it this way.

public Dictionary<string, List<string>> GetMapOfClassFunctions(string phpCode)
{
	string[] pCode = phpCode.Split();
	Dictionary<string, List<string>> result = new Dictinary<string, List<string>>();

	for(int i = 1; i < pCode.Length; i++)
	{
		
		if("<?php" == pCode[i])
		{
			for(i = i+1; i < pCode.Length -1 ; i++)
			{
				if(pCode[i] == "?>") break;
				else if(pCode[i] == "class")
				{
					
					i++;
					string className = pCode[i];
					result.Add(className, new List<string>());

					int parenthesis = 0;
					for(i = i+1; i < pCode.Length-1; i++)
					{
						if(pCode[i] == "{") parenthesis ++;
						else if(pCode[i] == "}" parenthesis--;
						else if(parenthesis == 0) break;
						else if(parenthesis == 1 && pCode[i] == "function")
						{
							i++;
							string functionName = pCode[i];
							result[className].Add(functionName);

							for( i = i + 1; i < pCode.Length; i++)
							{
								if(pCode[i] == "{") parenthesis ++;
								else if(pCode[i] == "}" parenthesis--;
								else if(parenthesis == 1) break;
							}
						}
					}
				}
			}
		}
	}
}

This is C++ version of solution:
I assume all the elements of php code is tokened.

#include<vector>
#include<list>
#include <map>
#include<string>
#include<iostream>
#include<fstream>

using namespace std;

map<string, list<string>* > * parse_php(vector<string> token)
{
  map <string, list<string>*> *result = new map<string, list<string>*>();
  map <string, list<string>*>::iterator found;
  bool inCode = false;
  bool inClass = false;
  int len = token.size();
  int parenthesis = 0;
  string currentClass;

  for (int i = 0; i < len; i++)
  {
    if (token[i] == "<?php" || token[i] == "<?")
      inCode = true;
    else if (token[i] == "?>")
      inCode = false;
    else if (inCode)
    {
      if (token[i] == "class")
      {
        inClass= true;
        if (i+1 < len)
        {
          currentClass = token[++i];
          list<string> *l = new list<string>();
          result->insert(pair<string, list<string>*>(currentClass, l));
          parenthesis = 0;
        }
      } else if (inClass) {
        if (token[i] == "{")
        {
          parenthesis++;
        }
        else if(token[i] == "}")
        {
          if (--parenthesis == 0)
          {
            inClass = false;
          }
        } else if (token[i] == "function")
       {
          if (i+1 <len)
          {
            found = result->find(currentClass);
            found->second->push_back(token[++i]);
          }
        }
      }
    }
  }
  return result;
}