Using InterSystems IRIS Natural Language Processing (NLP)
Blacklists
|
|
DomainGetOrCreate SET domn="mydomainwithbl" SET domId=##class(%iKnow.Domain).GetOrCreateId(domn) SET domoref=##class(%iKnow.Domain).%OpenId(domId) WRITE "The ",domn," domain with domain ID ",domId,! CreateBlackList SET blname="AviationBlacklist" SET blId=##class(%iKnow.Utils.MaintenanceAPI).CreateBlackList(domId,blname, "Aviation non-mechanical terms Blacklist") PopulateBlackList SET black=$LB("aircraft","airplane","flight","accident","event","incident","pilot", "student pilot","flight instructor","runway","accident site","ground","visibility","faa") SET ptr=0 FOR x=0:1:100 { SET moredata=$LISTNEXT(black,ptr,val) IF moredata=1 { SET stat=##class(%iKnow.Utils.MaintenanceAPI).AddStringToBlackList(domId,blId,val) } ELSE { WRITE x," entities in Blacklist",!! GOTO ListBlacklist } } ListBlacklist SET stat=##class(%iKnow.Utils.MaintenanceAPI).GetBlackLists(.bl,domId,0) SET i=1 WHILE $DATA(bl(i)) { WRITE $LISTTOSTRING(bl(i),",",1),! SET i=i+1 } WRITE "Printed the ",i-1," Blacklists",! SET stat=##class(%iKnow.Utils.MaintenanceAPI).GetBlackListElements(.ble,domId,blId) /* IF stat=1 {WRITE "success",!} ELSE {WRITE "GetBlackListElements failed" QUIT } */ SET j=1 WHILE $DATA(ble(j)) { WRITE $LISTTOSTRING(ble(j),",",1),! SET j=j+1 } WRITE "Printed the ",j-1," Blacklist elements",! CleanUp SET stat=##class(%iKnow.Utils.MaintenanceAPI).DropBlackList(domId,blId) IF stat=1 {WRITE "Blacklist deleted",!} ELSE {WRITE "DropBlackList failed" QUIT }
DomainGetOrCreate SET domn="mydomainwithbl" SET domId=##class(%iKnow.Domain).GetOrCreateId(domn) SET domoref=##class(%iKnow.Domain).%OpenId(domId) WRITE "The ",domn," domain with domain ID ",domId,! CreateBlackList1 SET blname="AviationTermsBlacklist" SET blId=##class(%iKnow.Utils.MaintenanceAPI).CreateBlackList(domId,blname, "Common aviation terms Blacklist") PopulateBlackList1 SET black=$LB("aircraft","airplane","flight","accident","event","incident","airport","runway") SET ptr=0 FOR x=0:1:100 { SET moredata=$LISTNEXT(black,ptr,val) IF moredata=1 { SET stat=##class(%iKnow.Utils.MaintenanceAPI).AddStringToBlackList(domId,blId,val) } } WRITE "Blacklist ",blname," populated",! CreateBlackList2 SET bl2name="JobTitleBlacklist" SET bl2Id=##class(%iKnow.Utils.MaintenanceAPI).CreateBlackList(0,bl2name, "Aviation personnel Blacklist") PopulateBlackList2 SET jobblack=$LB("pilot","copilot","student pilot","flight instructor","passenger") SET ptr=0 FOR x=0:1:100 { SET moredata=$LISTNEXT(jobblack,ptr,val) IF moredata=1 { SET stat=##class(%iKnow.Utils.MaintenanceAPI).AddStringToBlackList(0,bl2Id,val) } } WRITE "Blacklist ",bl2name," populated",!! ListBlacklists SET pIncludeCrossDomain=1 SET stat=##class(%iKnow.Utils.MaintenanceAPI).GetBlackLists(.bl,domId,pIncludeCrossDomain) SET i=1 WHILE $DATA(bl(i)) { IF $LIST(bl(i),1)<0 { WRITE "cross-domain:",!,$LISTTOSTRING(bl(i),",",1),! } ELSE { WRITE "domain-specific:",!,$LISTTOSTRING(bl(i),",",1),! } SET i=i+1 } WRITE "Printed the ",i-1," Blacklists",!! CleanUp SET stat=##class(%iKnow.Utils.MaintenanceAPI).DropBlackList(domId,blId) IF stat=1 {WRITE "domain Blacklist deleted",!} ELSE {WRITE "first DropBlackList failed" } SET stat=##class(%iKnow.Utils.MaintenanceAPI).DropBlackList(0,bl2Id) IF stat=1 {WRITE "cross-domain Blacklist deleted",!} ELSE {WRITE "second DropBlackList failed" }
#Include %IKPublic DomainCreateOrOpen SET dname="mydomain" IF (##class(%iKnow.Domain).Exists(dname)) { WRITE "The ",dname," domain already exists",! SET domoref=##class(%iKnow.Domain).Open(dname) GOTO DeleteOldData } ELSE { WRITE "The ",dname," domain does not exist",! SET domoref=##class(%iKnow.Domain).%New(dname) DO domoref.%Save() WRITE "Created the ",dname," domain with domain ID ",domoref.Id,! GOTO ListerAndLoader } DeleteOldData SET stat=domoref.DropData() IF stat { WRITE "Deleted the data from the ",dname," domain",!! GOTO ListerAndLoader } ELSE { WRITE "DropData error ",$System.Status.DisplayError(stat) QUIT} ListerAndLoader SET domId=domoref.Id SET flister=##class(%iKnow.Source.SQL.Lister).%New(domId) SET myloader=##class(%iKnow.Source.Loader).%New(domId) CreateBlackList1 SET blname="AviationTermsBlacklist" SET blId=##class(%iKnow.Utils.MaintenanceAPI).CreateBlackList(domId,blname, "Common aviation terms Blacklist") PopulateBlackList SET black=$LB("aircraft","airplane","flight","accident","event","incident","pilot","airport", "student pilot","flight instructor","runway","accident site","ground","visibility","faa") SET ptr=0 FOR x=0:1:100 { SET moredata=$LISTNEXT(black,ptr,val) IF moredata=1 { SET stat=##class(%iKnow.Utils.MaintenanceAPI).AddStringToBlackList(domId,blId,val) } } WRITE "Blacklist ",blname," populated",! QueryBuild SET myquery="SELECT TOP 100 ID AS UniqueVal,Type,NarrativeFull FROM Aviation.Event" SET idfld="UniqueVal" SET grpfld="Type" SET dataflds=$LB("NarrativeFull") UseLister SET stat=flister.AddListToBatch(myquery,idfld,grpfld,dataflds) IF stat '= 1 {WRITE "The lister failed: ",$System.Status.DisplayError(stat) QUIT } UseLoader SET stat=myloader.ProcessBatch() IF stat '= 1 {WRITE "The loader failed: ",$System.Status.DisplayError(stat) QUIT } SourceCountQuery SET numSrcD=##class(%iKnow.Queries.SourceQAPI).GetCountByDomain(domId) WRITE "The domain contains ",numSrcD," sources",! TopEntitiesQuery DO ##class(%iKnow.Queries.EntityAPI).GetTop(.result,domId,1,20,"",0,0,0,0,$LB(blId)) WRITE "NOTE: the ",blname," Blacklist",!, "has been applied to this list of top entities",! SET i=1 WHILE $DATA(result(i)) { SET outstr = $LISTTOSTRING(result(i),",",1) SET entity = $PIECE(outstr,",",2) SET freq = $PIECE(outstr,",",3) SET spread = $PIECE(outstr,",",4) WRITE "[",entity,"] appears ",freq," times in ",spread," sources",! SET i=i+1 } WRITE "Printed the top ",i-1," entities"
Content for this page loaded from GIKNOW_blacklist.xml on 2018-02-01 14:30:12
|