support either dropping levels or not, also based on if anything is f…

…iltered or not

Common/r_functionwhitelist.cpp

@@ -98,6 +98,7 @@ const std::set<std::string> R_FunctionWhiteList::functionWhiteList {
 	"lag",
 	"lapply",
 	"length",
+	"levels",
 	"library",
 	"list",
 	"local",

CommonData/column.cpp

@@ -1326,9 +1326,9 @@ stringvec Column::dataAsRLevels(intvec & values, const boolvec & filter, bool us
 
 	//At the end we make a mapping of the levels we have and need
 	//We make sure the map is up to date afterwards
-	for(int levelI=levels.size()-1; levelI >= 0; levelI--)
-		if(!levelsIncluded.count(levels[levelI]))
-			levels.erase(levels.begin() + levelI);
+	//for(int levelI=levels.size()-1; levelI >= 0; levelI--)
+	//	if(!levelsIncluded.count(levels[levelI]))
+	//		levels.erase(levels.begin() + levelI);
 
 	strintmap levelToValueMap;
 	for(size_t levelI=0; levelI<levels.size(); levelI++)

CommonData/databaseinterface.cpp

@@ -51,6 +51,9 @@ void DatabaseInterface::upgradeDBFromVersion(Version originalVersion)
 
 	if(originalVersion < "0.19.2" && !tableHasColumn("Filters", "name"))
 		runStatements("ALTER TABLE Filters  ADD COLUMN name		TEXT;");
+
+	if(originalVersion <= "0.19.3" && !tableHasColumn("Filters", "dropLevels"))
+		runStatements("ALTER TABLE Filters  ADD COLUMN dropLevels		INT;");
 
 	transactionWriteEnd();
 }
@@ -202,7 +205,7 @@ void DatabaseInterface::filterDelete(int filterIndex)
 }
 
 
-int DatabaseInterface::filterInsert(int dataSetId, const std::string & rFilter, const std::string & generatedFilter, const std::string & constructorJson, const std::string & constructorR, const std::string & name)
+int DatabaseInterface::filterInsert(int dataSetId, const std::string & rFilter, const std::string & generatedFilter, const std::string & constructorJson, const std::string & constructorR, const std::string & name, bool dropLevels)
 {
 	JASPTIMER_SCOPE(DatabaseInterface::filterInsertDataSet);
 	std::function<void(sqlite3_stmt *stmt)>  prepare = [&](sqlite3_stmt *stmt)
@@ -213,11 +216,12 @@ int DatabaseInterface::filterInsert(int dataSetId, const std::string & rFilter,
 		sqlite3_bind_text(stmt, 4, constructorJson.c_str(),		constructorJson.length(),	SQLITE_TRANSIENT);
 		sqlite3_bind_text(stmt, 5, constructorR.c_str(),		constructorR.length(),		SQLITE_TRANSIENT);
 		sqlite3_bind_text(stmt, 6, name.c_str(),				name.length(),				SQLITE_TRANSIENT);
+		sqlite3_bind_int(stmt,  7, int(dropLevels));
 	};
 
 	transactionWriteBegin();
 
-	int id = runStatementsId("INSERT INTO Filters (dataSet, rFilter, generatedFilter, constructorJson, constructorR, name) VALUES (?, ?, ?, ?, ?, ?) RETURNING rowid;", prepare);
+	int id = runStatementsId("INSERT INTO Filters (dataSet, rFilter, generatedFilter, constructorJson, constructorR, name, dropLevels) VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING rowid;", prepare);
 	runStatements("ALTER TABLE " + dataSetName(dataSetId) + " ADD " + filterTableName(id) +" INT NOT NULL DEFAULT 1;");
 
 	transactionWriteEnd();
@@ -314,7 +318,7 @@ bool DatabaseInterface::filterSelect(int filterIndex, boolvec & bools)
 	return changed;
 }
 
-void DatabaseInterface::filterUpdate(int filterIndex, const std::string & rFilter, const std::string & generatedFilter, const std::string & constructorJson, const std::string & constructorR, const std::string & name)
+void DatabaseInterface::filterUpdate(int filterIndex, const std::string & rFilter, const std::string & generatedFilter, const std::string & constructorJson, const std::string & constructorR, const std::string & name, bool dropLevels)
 {
 	JASPTIMER_SCOPE(DatabaseInterface::filterUpdate);
 	std::function<void(sqlite3_stmt *stmt)>  prepare = [&](sqlite3_stmt *stmt)
@@ -324,13 +328,14 @@ void DatabaseInterface::filterUpdate(int filterIndex, const std::string & rFilte
 		sqlite3_bind_text(stmt, 3, constructorJson.c_str(),	constructorJson.length(),	SQLITE_TRANSIENT);
 		sqlite3_bind_text(stmt, 4, constructorR.c_str(),	constructorR.length(),		SQLITE_TRANSIENT);
 		sqlite3_bind_text(stmt, 5, name.c_str(),			name.length(),				SQLITE_TRANSIENT);
-		sqlite3_bind_int (stmt,	6, filterIndex);
+		sqlite3_bind_int (stmt,	6, dropLevels);
+		sqlite3_bind_int (stmt,	7, filterIndex);
 	};
 
-	runStatements("UPDATE Filters SET rFilter=?, generatedFilter=?, constructorJson=?, constructorR=?, name=? WHERE id = ?;", prepare);
+	runStatements("UPDATE Filters SET rFilter=?, generatedFilter=?, constructorJson=?, constructorR=?, name=?, dropLevels=? WHERE id = ?;", prepare);
 }
 
-void DatabaseInterface::filterLoad(int filterIndex, std::string & rFilter, std::string & generatedFilter, std::string & constructorJson, std::string & constructorR, int & revision, std::string & name)
+void DatabaseInterface::filterLoad(int filterIndex, std::string & rFilter, std::string & generatedFilter, std::string & constructorJson, std::string & constructorR, int & revision, std::string & name, bool & dropLevels)
 {
 	JASPTIMER_SCOPE(DatabaseInterface::filterLoad);
 	std::function<void(sqlite3_stmt *stmt)>  prepare = [&](sqlite3_stmt *stmt)
@@ -342,16 +347,17 @@ void DatabaseInterface::filterLoad(int filterIndex, std::string & rFilter, std::
 	{
 		int colCount = sqlite3_column_count(stmt);
 
-		assert(colCount == 6);
+		assert(colCount == 7);
 		rFilter			= _wrap_sqlite3_column_text(stmt, 0);
 		generatedFilter	= _wrap_sqlite3_column_text(stmt, 1);
 		constructorJson	= _wrap_sqlite3_column_text(stmt, 2);
 		constructorR	= _wrap_sqlite3_column_text(stmt, 3);
 		revision		= sqlite3_column_int(		stmt, 4);
 		name			= _wrap_sqlite3_column_text(stmt, 5);
+		dropLevels		= sqlite3_column_int(		stmt, 6);
 	};
 
-	runStatements("SELECT rFilter, generatedFilter, constructorJson, constructorR, revision, name FROM Filters WHERE id = ?;", prepare, processRow);
+	runStatements("SELECT rFilter, generatedFilter, constructorJson, constructorR, revision, name, dropLevels FROM Filters WHERE id = ?;", prepare, processRow);
 }
 
 std::string DatabaseInterface::filterLoadErrorMsg(int filterIndex)

CommonData/databaseinterface.h

@@ -99,9 +99,9 @@ class DatabaseInterface
 	int			filterGetId(			const std::string & name);
 	bool		filterSelect(			int filterIndex,			boolvec & bools);																	///< Loads result and errorMsg and returns whether there was a change in either of those.
 	void		filterWrite(			int filterIndex,	const	boolvec & values);																	///< Overwrites the current filter values, no checks are done on the size. If too few the rest is TRUE nd superfluous bools are ignored.
-	int			filterInsert(			int dataSetId,		const std::string & rFilter = "", const std::string & generatedFilter = "", const std::string & constructorJson = "", const std::string & constructorR = "", const std::string & name = "");		///< Inserts a new Filter row into Filters and creates an empty FilterValues_#id. It returns id
-	void		filterUpdate(			int filterIndex,	const std::string & rFilter = "", const std::string & generatedFilter = "", const std::string & constructorJson = "", const std::string & constructorR = "", const std::string & name = "");		///< Updates an existing Filter row in Filters
-	void		filterLoad(				int filterIndex,		  std::string & rFilter,			std::string & generatedFilter,			  std::string & constructorJson,			std::string & constructorR, int & revision, std::string & name);			///< Loads an existing Filter row into arguments
+	int			filterInsert(			int dataSetId,		const std::string & rFilter = "", const std::string & generatedFilter = "", const std::string & constructorJson = "", const std::string & constructorR = "", const std::string & name = "", bool dropLevels=true);		///< Inserts a new Filter row into Filters and creates an empty FilterValues_#id. It returns id
+	void		filterUpdate(			int filterIndex,	const std::string & rFilter = "", const std::string & generatedFilter = "", const std::string & constructorJson = "", const std::string & constructorR = "", const std::string & name = "", bool dropLevels=true);		///< Updates an existing Filter row in Filters
+	void		filterLoad(				int filterIndex,		  std::string & rFilter,			std::string & generatedFilter,			  std::string & constructorJson,			std::string & constructorR, int & revision, std::string & name, bool & dropLevels);			///< Loads an existing Filter row into arguments
 	void		filterClear(			int filterIndex);																					///< Clears all values in Filter
 	void		filterDelete(			int filterIndex);
 	int			filterGetDataSetId(		int filterIndex);

CommonData/filter.cpp

@@ -18,10 +18,15 @@ Filter::Filter(DataSet * data, const std::string & name, bool createIfMissing)
 	else								throw std::runtime_error("Filter by name '" + _name + "' but it doesnt exist and createIfMissing=false!\nAre you sure this filter should exist?");
 }
 
+bool Filter::shouldDropLevels() const
+{
+	return dropLevels() && filteredRowCount() != data()->rowCount();
+}
+
 void Filter::dbCreate()
 {
 	assert(_id == -1);
-	_id = db().filterInsert(_data->id(), _rFilter, _generatedFilter, _constructorJson, _constructorR, _name);
+	_id = db().filterInsert(_data->id(), _rFilter, _generatedFilter, _constructorJson, _constructorR, _name, _dropLevels);
 }
 
 void Filter::dbUpdate()
@@ -32,7 +37,7 @@ void Filter::dbUpdate()
 
 	db().transactionWriteBegin();
 	if(!_data->writeBatchedToDB())
-		db().filterUpdate(_id, _rFilter, _generatedFilter, _constructorJson, _constructorR, _name);
+		db().filterUpdate(_id, _rFilter, _generatedFilter, _constructorJson, _constructorR, _name, _dropLevels);
 
 	incRevision();
 	db().transactionWriteEnd();
@@ -59,7 +64,7 @@ void Filter::dbLoad()
 	db().transactionReadBegin();
 
 	std::string nameInDB = "";
-	db().filterLoad(_id, _rFilter, _generatedFilter, _constructorJson, _constructorR, _revision, nameInDB);
+	db().filterLoad(_id, _rFilter, _generatedFilter, _constructorJson, _constructorR, _revision, nameInDB, _dropLevels);
 	assert(nameInDB == _name);
 
 	_filteredRowCount	= 0;
@@ -111,6 +116,12 @@ void Filter::setFilterValueNoDB(size_t row, bool val)
 	_filtered[row] = val;
 }
 
+void Filter::setDropLevels(bool dropEm)
+{
+	_dropLevels = dropEm;
+	dbUpdate();
+}
+
 void Filter::setRowCount(size_t rows)
 {
 	_filtered.resize(rows);

CommonData/filter.h

@@ -26,6 +26,8 @@ class Filter : public DataSetBaseNode
 
 	DataSet					*	data()				const { return _data;					}
 	int							id()				const { return _id;						}
+	bool						dropLevels()		const { return _dropLevels;				}
+	bool						shouldDropLevels()	const;
 	const std::string		&	name()				const { return _name;					}
 	const std::string		&	rFilter()			const { return _rFilter;				}
 	const std::string		&	generatedFilter()	const { return _generatedFilter;		}
@@ -43,6 +45,7 @@ class Filter : public DataSetBaseNode
 	void				setName(			const std::string	& name)				{ _name				= name;				dbUpdate(); }
 	bool				setFilterVector(	const boolvec		& filterResult);
 	void				setFilterValueNoDB(	size_t	row, bool val);
+	void				setDropLevels(		bool	dropEm);
 	void				setRowCount(		size_t	rows);
 	void				setId(				int		id)			{ _id = id; }
 
@@ -72,6 +75,7 @@ class Filter : public DataSetBaseNode
 							_constructorR		= "",
 							_errorMsg			= "",
 							_name				= "";
+	bool					_dropLevels			= true; 
 	std::vector<bool>		_filtered;
 };
 

CommonData/internalDbDefinition.sql

@@ -21,6 +21,7 @@ CREATE TABLE Filters (
 	constructorR	TEXT, 
 	errorMsg		TEXT,
 	revision		INT DEFAULT 0, 
+	dropLevels		INT,
 
 	FOREIGN KEY(dataSet) REFERENCES DataSets(id)
 );

Desktop/data/columnmodel.cpp

@@ -801,6 +801,7 @@ void ColumnModel::_deleteLabel(int labelIndex)
 		return;
 
 	column()->labelsRemove(labelIndex);
+	column()->incRevision();
 	refresh();
 	DataSetPackage::pkg()->emitColumnChanged(columnNameQ());
 }

Engine/engine.cpp

@@ -370,10 +370,12 @@ void Engine::updateOptionsAccordingToMeta(Json::Value & encodedOptions)
 void Engine::runFilter(const std::string & filter, const std::string & generatedFilter, int filterRequestId)
 {
 	try
-	{
-		std::string strippedFilter		= stringUtils::stripRComments(filter);
-		std::vector<bool> filterResult	= rbridge_applyFilter(strippedFilter, generatedFilter);
-		std::string RPossibleWarning	= jaspRCPP_getLastErrorMsg();
+	{		
+
+
+		std::string		strippedFilter		= stringUtils::stripRComments(filter);
+		boolvec			filterResult		= rbridge_applyFilter(strippedFilter, generatedFilter);
+		std::string		RPossibleWarning	= jaspRCPP_getLastErrorMsg();
 
 		Log::log() << "Engine::runFilter ran:\n\t" << strippedFilter << "\n\tRPossibleWarning='" << RPossibleWarning << "'\n\t\tfor revision " << _dataSet->filter()->revision() << std::endl;
 
@@ -455,6 +457,8 @@ void Engine::receiveRCodeMessage(const Json::Value & jsonRequest)
 // Evaluating arbitrary R code (as string) which returns a string
 void Engine::runRCode(const std::string & rCode, int rCodeRequestId, bool whiteListed)
 {
+	provideAndUpdateDataSet();
+
 
 	std::string rCodeResult = whiteListed ? rbridge_evalRCodeWhiteListed(rCode.c_str(), true) : jaspRCPP_evalRCode(rCode.c_str(), true);
 
@@ -475,6 +479,8 @@ void Engine::runRCodeCommander(std::string rCode)
 
 	if(thereIsSomeData)
 	{
+
+
 		rCode = ColumnEncoder::encodeAll(rCode);
 		jaspRCPP_runScript((rCmdDataName + "<- .readFullDatasetToEnd();").c_str());
 		jaspRCPP_runScript((rCmdFiltered + "<- .readFullFilteredDatasetToEnd();").c_str());
@@ -558,6 +564,8 @@ void Engine::runComputeColumn(const std::string & computeColumnName, const std::
 		{
 			std::string computeColumnNameEnc = ColumnEncoder::columnEncoder()->encode(computeColumnName);
 			computeColumnResponse["columnName"]		= computeColumnNameEnc;
+
+
 
 			std::string computeColumnResultStr		= rbridge_evalRComputedColumn(
 						computeColumnCode, 
@@ -739,7 +747,7 @@ void Engine::runAnalysis()
 	default:	break;
 	}
 
-	   provideAndUpdateDataSet();
+	provideAndUpdateDataSet();
 	Log::log() << "Analysis will be run now." << std::endl;
 
 	Json::Value encodedAnalysisOptions = _analysisOptions;

Engine/enginebase.cpp

@@ -86,6 +86,9 @@ DataSet * EngineBase::provideAndUpdateDataSet()
 
 	if(_dataSet && setColumnNames)
 		ColumnEncoder::columnEncoder()->setCurrentNames(_dataSet->getColumnNames(), true);
+
+	if(_dataSet)
+		jaspRCPP_setShouldDropLevels(_dataSet->filter()->shouldDropLevels());
 
 	JASPTIMER_STOP(EngineBase::provideAndUpdateDataSet());
 

Engine/rbridge.cpp

@@ -453,7 +453,7 @@ void __freeRBridgeColumnType(RBridgeColumnType *columns, size_t colMax)
 	for (int i = 0; i < colMax; i++)
 		free(columns[i].name);
 
-	free(columns);
+	delete[](columns);
 }
 
 extern "C" RBridgeColumn* STDCALL rbridge_readDataSetRequested(size_t * colMax, bool obeyFilter)
@@ -704,9 +704,20 @@ void rbridge_setupRCodeEnv(int rowCount, const std::string & dataname)
 	setupFilterEnv =	"rowcount    <- " + std::to_string(rowCount) +  ";";
 	jaspRCPP_runScript(setupFilterEnv.c_str());
 
+	rbridge_setupRCodeEnvReadData(dataname, ".readCompColDatasetToEnd()");
+}
+
+void rbridge_setupRCodeFilterEnv(int rowCount, const std::string & dataname)
+{
+	static std::string setupFilterEnv;
+
+	setupFilterEnv =	"rowcount    <- " + std::to_string(rowCount) +  ";";
+	jaspRCPP_runScript(setupFilterEnv.c_str());
+
 	rbridge_setupRCodeEnvReadData(dataname, ".readFilterDatasetToEnd()");
 }
 
+
 void rbridge_setupRCodeEnvReadData(const std::string & dataname, const std::string & readFunction)
 {
 	static std::string setupFilterEnv;
@@ -746,7 +757,7 @@ std::vector<bool> rbridge_applyFilter(const std::string & filterCode, const std:
 
 	bool * arrayPointer = nullptr;
 
-	rbridge_setupRCodeEnv(rowCount);
+	rbridge_setupRCodeFilterEnv(rowCount);
 	int arrayLength	= jaspRCPP_runFilter(filter64.c_str(), &arrayPointer);
 	rbridge_detachRCodeEnv();
 

Engine/rbridge.h

@@ -83,9 +83,10 @@ extern "C" {
 
 	std::string rbridge_runModuleCall(const std::string &name, const std::string &title, const std::string &moduleCall, const std::string &dataKey, const std::string &options, const std::string &stateKey, int analysisID, int analysisRevision, bool developerMode, ColumnEncoder::colsPlusTypes datasetColsTypes, bool preloadData);
 
-	void	rbridge_setupRCodeEnvReadData(const std::string & dataname, const std::string & readFunction);
-	void	rbridge_setupRCodeEnv(int rowCount, const std::string & dataname = "data");
-	void	rbridge_detachRCodeEnv(				const std::string & dataname = "data");
+	void	rbridge_setupRCodeEnvReadData(				const std::string & dataname, const std::string & readFunction);
+	void	rbridge_setupRCodeEnv(		int rowCount,	const std::string & dataname = "data");
+	void	rbridge_setupRCodeFilterEnv(int rowCount,	const std::string & dataname = "data");
+	void	rbridge_detachRCodeEnv(						const std::string & dataname = "data");
 
 	void freeRBridgeColumns();
 	void freeRBridgeColumnDescription(RBridgeColumnDescription* columns, size_t colMax);