Source: site.view [edit]
Function name: indexNews
Arguments:
Description:
Page type: webl
Render function:   fullPageRender
Module: geneticFinance

Page source:

// Many of the company names have things like "-CLASS A SHARES" in the name, prune those.
var CleanField = fun(s)

   var i = Str_IndexOf("-", s);
   if (i > 0) then
      s = Select(s, 0, i)
   end;
   return s;
end;
  

// We need it in this format: 20050101:20050201  
var FormatDate = fun(dt)
   var bf = dt;
   var sdf = Wub_SimpleDateFormat("dd-MMM-yy");
   var d = sdf.parse(dt) ? nil;
   if d != nil then
      sdf = Wub_SimpleDateFormat("YYYYMMdd");
      dt = sdf.format(d);
      var c = Wub_GetCalendarClass().getInstance();
      c.setTime(d);
      var CalendarDATE = 5;
      c.add(CalendarDATE, -1);  // number of days to add
      bf = sdf.format(c.getTime());
      bf = bf;
   end;
   return [. day=dt, dayBefore=bf .];
end;

var readCSV = fun(cmd, idCol)

   var res = [. .];
   var headers = nil;
   every line in Str_Split(WubCall(cmd, []), "\n") do
      var cols = Str_Split(line, ",");
      if headers == nil then
         headers = cols
      else
         var obj = [. .];
         var i = 0;
         every h in headers do
            obj[h] := CleanField(cols[i]);
            i = i + 1
         end;
         res[Str_ToLowerCase(cols[idCol])] := obj
      end;
   end;
   return res
end;

var errs = [];

var main = fun()
   var stockInfo = readCSV("stockInfo", 2);

   var names=[. .];

   every line in Str_Split(WubCall("stockChange", []), "\n") do
      var cols = Str_Split(line, ",");
      var symbol = Str_ToLowerCase(cols[1]);
      var dt = FormatDate(cols[0]);
      var change = ToReal(cols[2]);
      (names[symbol] := [. fullName=stockInfo[symbol].company, change=change, dt=dt .] ) ? (errs = errs + [symbol]);
   end;
   
   var i = 0;
   var res = [];
   
   var ok = true;

   Wub_DropDB("newsIndex");
   var db = Wub_GetDB("newsIndex");
   var coll = Wub_GetCollection(db, "articles");


// To query the database, create a query object and call the QueryDB function:
// var dbobj = Wub_NewDBObject([. name="Adam Cheyer" .]); var res = Wub_QueryDB(coll, dbobj);
   
   
   var j = 0;
   every symbol in names do
      if i < 20000 then
          var news = WubCall("searchDates", [ names[symbol].fullName + " (" + symbol + ")", 
                                       names[symbol].dt.day + ":" + names[symbol].dt.day]) ? [];
          every n in news do
                j = j + 1;
				var obj = [. num=ToString(j), symbol=symbol, fullname=names[symbol].fullName, day=names[symbol].dt.day,
       						 offset="0", score="?", text="", title=n.title, sitename=n.sitename, link=n.link, 
                             snippet=n.snippet, cacheid=n.cacheId .];
                var dbobj = Wub_NewDBObject(obj); 
                Wub_InsertDB(coll, dbobj);

          end;
   
          news = WubCall("searchDates", [ names[symbol].fullName + " (" + symbol + ")", 
                                       names[symbol].dt.dayBefore + ":" + names[symbol].dt.dayBefore]) ? [];
          every n in news do
                j = j + 1;
				var obj = [. num=ToString(j), symbol=symbol, fullname=names[symbol].fullName, day=names[symbol].dt.dayBefore,
       						 offset="-1", score="?", text="", title=n.title, sitename=n.sitename, link=n.link, 
                             snippet=n.snippet, cacheid=n.cacheId .];
                var dbobj = Wub_NewDBObject(obj); 
                Wub_InsertDB(coll, dbobj);

          end

      end;
      i = i + 1
   end;
   "Found " + ToString(j) + " news articles about " + ToString(i) + " companies"
end;
  
main();