Finally got precompiled headers to work.

Now all C++ files need to #include <util/prec.hpp>
 That is why all .cpp files are touched by this commit

Many changes to installers and update checking:
     - the window is now called PackagesWindow, in a new source file
     - update checking is now independent from the PackagesWindow. For update checking only a list of package versions are needed (vector<PackageDependency>). This is much less information to download at each startup.
     - the list of available packages is now a list of available Installers, since an installer can contain multiple packages.
     - moved the logic of dependency checking etc. to data/installer
     - moved the actual installation to util/io/package_manager
     - moved directory iteration/creation logic to util/file_utils
     - added PackageDirectory: the local and global package directory now have their own object (was part of PackageManager)
     - added PackageVersion: for detecting if a package has been modified after it was installed.
     - added PackageDescription: description/header of a package. Basicly the same as what Packaged provides.
     - added DownloadableInstaller: where to find an insaller, what does it contain?
     - added InstallablePackage: brining it all together: installer, package, status, action.

Current status: the insaller is currently broken in a few places, more on that soon.

git-svn-id: svn://svn.code.sf.net/p/magicseteditor/code/trunk@792 0fc631ac-6414-0410-93d0-97cfa31319b6

src/util/action_stack.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/action_stack.hpp>
 #include <util/for_each.hpp>
 #include <algorithm>

src/util/age.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/age.hpp>
 
 // ----------------------------------------------------------------------------- : Age

src/util/alignment.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/alignment.hpp>
 #include <util/reflect.hpp>
 

src/util/atomic.hpp

@@ -16,7 +16,7 @@
 
 // ----------------------------------------------------------------------------- : AtomicInt : windows
 
-#ifdef __WXMSW__
+#if defined(__WXMSW__)
 	
 	#ifdef _MSC_VER
 		extern "C" {
@@ -28,7 +28,7 @@
 		#pragma intrinsic (_InterlockedDecrement)
 		#define InterlockedDecrement _InterlockedDecrement
 	#endif
-		
+	
 	/// An integer which is equivalent to an AtomicInt, but which doesn't support attomic operations
 	typedef LONG AtomicIntEquiv;
 	
@@ -50,7 +50,33 @@
 	  private:
 		AtomicIntEquiv    v;  ///< The value
 	};
-
+	
+	/// We have a fast AtomicInt
+	#define HAVE_FAST_ATOMIC
+	
+// ----------------------------------------------------------------------------- : AtomicInt : GCC
+#elif defined(__GNUC__)
+	
+	/// An integer which is equivalent to an AtomicInt, but which doesn't support attomic operations
+	typedef unsigned int AtomicIntEquiv;
+	
+	/// An integer that can be incremented and decremented atomicly
+	class AtomicIntEquiv {
+	  public:
+		AtomicIntEquiv(AtomicIntEquiv v) : v(v) {}
+		inline operator AtomicIntEquiv() const {
+			return v;
+		}
+		inline AtomicIntEquiv operator ++ () {
+			return __sync_add_and_fetch(&v,1);
+		}
+		inline AtomicIntEquiv operator -- () {
+			return __sync_add_and_fetch(&v,(AtomicIntEquiv)-1);
+		}
+	  private:
+		AtomicIntEquiv v;
+	};
+	
 	/// We have a fast AtomicInt
 	#define HAVE_FAST_ATOMIC
 	

src/util/delayed_index_maps.hpp

@@ -58,7 +58,11 @@ void Reader::handle(DelayedIndexMapsData<Key,Value>& d) {
 }
 template <typename Key, typename Value>
 void Writer::handle(const DelayedIndexMapsData<Key,Value>& d) {
-	handle(d.read_data);
+	if (!d.unread_data.empty()) {
+		handle(d.unread_data); // TODO: how to handle filenames
+	} else {
+		handle(d.read_data);
+	}
 }
 template <typename Key, typename Value>
 void GetMember::handle(const DelayedIndexMapsData<Key,Value>& d) {

src/util/error.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/error.hpp>
 
 DECLARE_TYPEOF_COLLECTION(ScriptParseError);

src/util/file_utils.cpp

@@ -0,0 +1,134 @@
+//+----------------------------------------------------------------------------+
+//| Description:  Magic Set Editor - Program to make Magic (tm) cards          |
+//| Copyright:    (C) 2001 - 2007 Twan van Laarhoven                           |
+//| License:      GNU General Public License 2 or later (see file COPYING)     |
+//+----------------------------------------------------------------------------+
+
+// ----------------------------------------------------------------------------- : Includes
+
+#include <util/prec.hpp>
+#include <util/file_utils.hpp>
+#include <wx/filename.h>
+#include <wx/dir.h>
+
+DECLARE_TYPEOF_COLLECTION(String);
+
+// ----------------------------------------------------------------------------- : File names
+
+String normalize_filename(const String& name) {
+	wxFileName fn(name);
+	fn.Normalize();
+	return fn.GetFullPath();
+}
+
+String normalize_internal_filename(const String& name) {
+	String ret;
+	FOR_EACH_CONST(c, name) {
+		if (c==_('\\')) ret += _('/');
+		else            ret += toLower(c);
+	}
+	return ret;
+}
+
+bool ignore_file(const String& name) {
+	// Files that are never part of a package,
+	// i.e. random stuff the OS file manager dumps without being asked
+	return name == _("Thumbs.db"); // winXP explorer thumbnails
+}
+
+// ----------------------------------------------------------------------------- : Directories
+
+bool create_parent_dirs(const String& file) {
+	for (size_t pos = file.find_first_of(_("\\/")) ;
+	     pos != String::npos ;
+	     pos = file.find_first_of(_("\\/"),pos+1)) {
+		String part = file.substr(0,pos);
+		if (!wxDirExists(part)) {
+			if (!wxMkdir(part)) return false;
+		}
+	}
+	return true;
+}
+
+// ----------------------------------------------------------------------------- : Removing
+
+class RecursiveDeleter : public wxDirTraverser {
+  public:
+	RecursiveDeleter(const String& start) {
+		to_delete.push_back(start);
+	}
+	~RecursiveDeleter() {
+		FOR_EACH_REVERSE(dir, to_delete) {
+			wxRmdir(dir);
+		}
+	}
+	
+	wxDirTraverseResult OnFile(const String& filename) {
+		if (!wxRemoveFile(filename))
+			handle_error(_("Cannot delete ") + filename + _(". ")
+				_("The remainder of the package has still been removed, if possible.")
+				_("Other packages may have been removed, including packages that this on is dependent on. Please remove manually."));
+		return wxDIR_CONTINUE;
+	}
+	wxDirTraverseResult OnDir(const String& dirname) {
+		to_delete.push_back(dirname);
+		return wxDIR_CONTINUE;
+	}
+  private:
+	vector<String> to_delete;
+};
+
+bool remove_file_or_dir(const String& name) {
+	if (wxFileExists(name)) {
+		return wxRemoveFile(name);
+	} else if (wxDirExists(name)) {
+		wxDir dir(name);
+		RecursiveDeleter rd(name);
+		dir.Traverse(rd);
+		return true;
+	} else {
+		return false;
+	}
+}
+
+// ----------------------------------------------------------------------------- : Renaming
+
+bool rename_file_or_dir(const String& from, const String& to) {
+	create_parent_dirs(to);
+	return wxRenameFile(from, to);
+}
+
+// ----------------------------------------------------------------------------- : Moving
+
+class IgnoredMover : public wxDirTraverser {
+  public:
+	IgnoredMover(const String& from, const String& to)
+		: from(from), to(to)
+	{}
+	wxDirTraverseResult OnFile(const String& filename) {
+		tryMove(filename);
+		return wxDIR_CONTINUE;
+	}
+	wxDirTraverseResult OnDir(const String& dirname) {
+		return tryMove(dirname) ? wxDIR_IGNORE : wxDIR_CONTINUE;
+	}
+  private:
+	String from, to;
+	bool tryMove(const String& from_path) {
+		if (is_substr(from_path,0,from)) {
+			String to_path = to + from_path.substr(from.size());
+			return rename_file_or_dir(from_path, to_path);
+		} else {
+			// This shouldn't happen
+			return false;
+		}
+	}
+};
+
+void move_ignored_files(const String& from_dir, const String& to_dir) {
+	if (wxDirExists(from_dir) && wxDirExists(to_dir)) {
+		wxDir dir(from_dir);
+		IgnoredMover im(from_dir, to_dir);
+		dir.Traverse(im);
+	}
+}

src/util/file_utils.hpp

@@ -0,0 +1,44 @@
+//+----------------------------------------------------------------------------+
+//| Description:  Magic Set Editor - Program to make Magic (tm) cards          |
+//| Copyright:    (C) 2001 - 2007 Twan van Laarhoven                           |
+//| License:      GNU General Public License 2 or later (see file COPYING)     |
+//+----------------------------------------------------------------------------+
+
+#ifndef HEADER_UTIL_FILE_UTILS
+#define HEADER_UTIL_FILE_UTILS
+
+// ----------------------------------------------------------------------------- : Includes
+
+#include <util/prec.hpp>
+
+// ----------------------------------------------------------------------------- : File names
+
+/// Normalize a filename as much as possible, for real files
+String normalize_filename(const String& filename);
+
+/// Normalize a filename as much as possible, for files in packages
+String normalize_internal_filename(const String& filename);
+
+/// Should a file with the given name be ignored?
+bool ignore_file(const String& name);
+
+// ----------------------------------------------------------------------------- : Removing and renaming
+
+/// Ensure that the parent directories of the given filename exist
+bool create_parent_dirs(const String& file);
+
+/// Remove the given file or directory
+/** It is not an error if the file doesn't exist.
+ *  Removes all files in a directory.
+ *  Returns true if something was removed
+ */
+bool remove_file_or_dir(const String& file);
+
+/// Rename a file or directory
+bool rename_file_or_dir(const String& old_name, const String& new_name);
+
+/// Move files/dirs that are ignored by packages to another directory
+void move_ignored_files(const String& from_dir, const String& to_dir);
+
+// ----------------------------------------------------------------------------- : EOF
+#endif

src/util/io/get_member.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/io/get_member.hpp>
 #include <util/vector2d.hpp>
 #include <script/script.hpp>

src/util/io/get_member.hpp

@@ -29,6 +29,7 @@ class GetDefaultMember {
 	inline bool scripting() const { return true; }
 	inline bool isComplex() const { return false; }
 	inline void addAlias(int, const Char*, const Char*) {}
+	inline void handleIgnore(int, const Char*) {}
 	inline void handleAppVersion() {} // no effect
 	
 	/// The result, or script_nil if the member was not found
@@ -76,6 +77,7 @@ class GetMember : private GetDefaultMember {
 	inline bool scripting() const { return true; }
 	inline bool isComplex() const { return false; }
 	inline void addAlias(int, const Char*, const Char*) {}
+	inline void handleIgnore(int, const Char*) {}
 	inline void handleAppVersion() {} // no effect
 	
 	/// The result, or script_nil if the member was not found

src/util/io/package.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/io/package.hpp>
 #include <util/io/package_manager.hpp>
 #include <util/error.hpp>
@@ -180,7 +181,7 @@ InputStreamP Package::openIn(const String& file) {
 		Packaged* p = dynamic_cast<Packaged*>(this);
 		return packages.openFileFromPackage(p, file);
 	}
-	FileInfos::iterator it = files.find(toStandardName(file));
+	FileInfos::iterator it = files.find(normalize_internal_filename(file));
 	if (it == files.end()) {
 		// does it look like a relative filename?
 		if (filename.find(_(".mse-")) != String::npos) {
@@ -262,7 +263,7 @@ void Package::referenceFile(const String& file) {
 
 String Package::absoluteName(const String& file) {
 	assert(wxThread::IsMain());
-	FileInfos::iterator it = files.find(toStandardName(file));
+	FileInfos::iterator it = files.find(normalize_internal_filename(file));
 	if (it == files.end()) {
 		throw FileNotFoundError(file, filename);
 	}
@@ -307,7 +308,7 @@ void Package::loadZipStream() {
 	while (true) {
 		wxZipEntry* entry = zipStream->GetNextEntry();
 		if (!entry) break;
-		String name = toStandardName(entry->GetName());
+		String name = normalize_internal_filename(entry->GetName());
 		files[name].zipEntry = entry;
 	}
 	zipStream->CloseEntry();
@@ -323,6 +324,7 @@ void Package::openSubdir(const String& name) {
 	// find files
 	String f; // filename
 	for(bool ok = d.GetFirst(&f, wxEmptyString, wxDIR_FILES | wxDIR_HIDDEN) ; ok ; ok = d.GetNext(&f)) {
+		if (ignore_file(f)) continue;
 		// add file
 		addFile(name + f);
 		// get modified time
@@ -422,18 +424,22 @@ void Package::saveToZipfile(const String& saveAs, bool remove_unused) {
 
 
 Package::FileInfos::iterator Package::addFile(const String& name) {
-	return files.insert(make_pair(toStandardName(name), FileInfo())).first;
+	return files.insert(make_pair(normalize_internal_filename(name), FileInfo())).first;
 }
 
-String Package::toStandardName(const String& name) {
-	String ret;
-	FOR_EACH_CONST(c, name) {
-		if (c==_('\\')) ret += _('/');
-		else            ret += toLower(c);
+DateTime Package::modificationTime(const pair<String, FileInfo>& fi) const {
+	if (fi.second.wasWritten()) {
+		return wxFileName(fi.first).GetModificationTime();
+	} else if (fi.second.zipEntry) {
+		return fi.second.zipEntry->GetDateTime();
+	} else if (wxFileExists(filename+_("/")+fi.first)) {
+		return wxFileName(filename+_("/")+fi.first).GetModificationTime();
+	} else {
+		return DateTime();
 	}
-	return ret;
 }
 
+
 // ----------------------------------------------------------------------------- : Packaged
 
 template <> void Reader::handle(PackageDependency& dep) {
@@ -463,6 +469,7 @@ IMPLEMENT_REFLECTION(Packaged) {
 	REFLECT(full_name);
 	REFLECT_N("icon", icon_filename);
 	REFLECT_NO_SCRIPT(position_hint);
+	REFLECT(installer_group);
 	REFLECT(version);
 	REFLECT(compatible_version);
 	REFLECT_NO_SCRIPT_N("depends ons", dependencies); // hack for singular_form

src/util/io/package.hpp

@@ -12,6 +12,7 @@
 #include <util/reflect.hpp>
 #include <util/dynamic_arg.hpp>
 #include <util/error.hpp>
+#include <util/file_utils.hpp>
 
 class Package;
 class wxFileInputStream;
@@ -147,8 +148,8 @@ class Package : public IntrusivePtrVirtualBase {
 		bool keep;               ///< Should this file be kept in the package? (as opposed to deleting it)
 		String tempName;         ///< Name of the temporary file where new contents of this file are placed
 		wxZipEntry* zipEntry;    ///< Entry in the zip file for this file
-		inline bool wasWritten() ///< Is this file changed, and therefore written to a temporary file?
-		 { return !tempName.empty(); }
+		/// Is this file changed, and therefore written to a temporary file?
+		inline bool wasWritten() const { return !tempName.empty(); }
 	};
 		
 	/// Filename of the package
@@ -160,6 +161,8 @@ class Package : public IntrusivePtrVirtualBase {
 	/** Note: must be public for DECLARE_TYPEOF to work */
 	typedef map<String, FileInfo> FileInfos;
 	inline const FileInfos& getFileInfos() const { return files; }
+	/// When was a file last modified?
+	DateTime modificationTime(const pair<String, FileInfo>& fi) const;
   private:
 	/// All files in the package
 	FileInfos files;
@@ -175,7 +178,6 @@ class Package : public IntrusivePtrVirtualBase {
 	void saveToZipfile(const String&, bool);
 	void saveToDirectory(const String&, bool);
 	FileInfos::iterator addFile(const String& file);
-	static String toStandardName(const String& file);
 };
 
 // ----------------------------------------------------------------------------- : Packaged
@@ -199,6 +201,7 @@ class Packaged : public Package {
 	
 	Version version;			///< Version number of this package
 	Version compatible_version;	///< Earliest version number this package is compatible with
+	String installer_group;		///< Group to place this package in in the installer
 	String short_name;			///< Short name of this package
 	String full_name;			///< Name of this package, for menus etc.
 	String icon_filename;		///< Filename of icon to use in package lists

src/util/io/package_manager.cpp

@@ -6,66 +6,58 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/io/package_manager.hpp>
 #include <util/error.hpp>
+#include <util/file_utils.hpp>
 #include <data/game.hpp>
 #include <data/stylesheet.hpp>
 #include <data/symbol_font.hpp>
 #include <data/locale.hpp>
 #include <data/export_template.hpp>
+#include <data/installer.hpp>
 #include <wx/stdpaths.h>
+#include <wx/wfstream.h>
 
-// ----------------------------------------------------------------------------- : PackageManager
+DECLARE_TYPEOF_COLLECTION(InstallablePackageP);
+DECLARE_TYPEOF_COLLECTION(PackageVersion::FileInfo);
+
+// ----------------------------------------------------------------------------- : PackageManager : in memory
 
 PackageManager packages;
 
 
 void PackageManager::init() {
-	// determine data directory
-	global_data_directory = wxStandardPaths::Get().GetDataDir();
-	// check if this is the actual data directory, especially during debugging,
-	// the data may be higher up:
-	//  exe path  = mse/build/debug/mse.exe
-	//  data path = mse/data
-	while (!wxDirExists(global_data_directory + _("/data"))) {
-		String d = global_data_directory;
-		global_data_directory = wxPathOnly(global_data_directory);
-		if (d == global_data_directory) {
-			// we are at the root -> 'data' not found anywhere in the path -> fatal error
-			throw Error(_("The global MSE data files can not be found, there should be a directory called 'data' with these files. The expected directory to find it in was ") + wxStandardPaths::Get().GetDataDir());
-		}
-	}
-	global_data_directory += _("/data");
-	// It's not an error for the local directory not to exist.
-	local_data_directory = wxStandardPaths::Get().GetUserDataDir();
-	local_data_directory += _("/data");
+	local.init(true);
+	global.init(false);
+}
+void PackageManager::destroy() {
+	loaded_packages.clear();
+}
+void PackageManager::reset() {
+	loaded_packages.clear();
 }
 
-PackagedP PackageManager::openAny(const String& name, bool just_header) {
+PackagedP PackageManager::openAny(const String& name_, bool just_header) {
+	String name = trim(name_);
 	// Attempt to load local data first.
 	String filename;
-	wxFileName fn;
 	if (wxFileName(name).IsRelative()) {
 		// local data dir?
-		fn.Assign(local_data_directory + _("/") + name);
-		fn.Normalize();
-		filename = fn.GetFullPath();
+		filename = normalize_filename(local.name(name));
 		if (!wxFileExists(filename) && !wxDirExists(filename)) {
 			// global data dir
-			fn.Assign(global_data_directory + _("/") + name);
-			fn.Normalize();
-			filename = fn.GetFullPath();
+			filename = normalize_filename(global.name(name));
 		}
 	} else { // Absolute filename
-		fn.Assign(name);
-		fn.Normalize();
-		filename = fn.GetFullPath();
+		filename = normalize_filename(name);
 	}
 
 	// Is this package already loaded?
 	PackagedP& p = loaded_packages[filename];
 	if (!p) {
 		// load with the right type, based on extension
+		wxFileName fn(filename);
 		if      (fn.GetExt() == _("mse-game"))            p = new_intrusive<Game>();
 		else if (fn.GetExt() == _("mse-style"))           p = new_intrusive<StyleSheet>();
 		else if (fn.GetExt() == _("mse-locale"))          p = new_intrusive<Locale>();
@@ -76,22 +68,21 @@ PackagedP PackageManager::openAny(const String& name, bool just_header) {
 			throw PackageError(_("Unrecognized package type: '") + fn.GetExt() + _("'\nwhile trying to open: ") + name);
 		}
 		p->open(filename, just_header);
+	} else if (!just_header) {
+		p->loadFully();
 	}
 	return p;
 }
 
 void PackageManager::findMatching(const String& pattern, vector<PackagedP>& out) {
-	String file;
 	// first find local packages
-	if (wxDirExists(local_data_directory)) {
-		String file = wxFindFirstFile(local_data_directory + _("/") + pattern, 0);
-		while (!file.empty()) {
-			out.push_back(openAny(file, true));
-			file = wxFindNextFile();
-		}
+	String file = local.findFirstMatching(pattern);
+	while (!file.empty()) {
+		out.push_back(openAny(file, true));
+		file = wxFindNextFile();
 	}
 	// then global packages not already in the list
-	file = wxFindFirstFile(global_data_directory + _("/") + pattern, 0);
+	file = global.findFirstMatching(pattern);
 	while (!file.empty()) {
 		PackagedP p = openAny(file, true);
 		if (find(out.begin(), out.end(), p) == out.end()) {
@@ -121,17 +112,21 @@ InputStreamP PackageManager::openFileFromPackage(Packaged*& package, const Strin
 	throw FileNotFoundError(name, _("No package name specified, use '/package/filename'"));
 }
 
+// ----------------------------------------------------------------------------- : PackageManager : on disk
+
 bool PackageManager::checkDependency(const PackageDependency& dep, bool report_errors) {
-	// try local package
-	String name = local_data_directory + _("/") + dep.package;
-	if (!wxFileExists(name) && !wxDirExists(name)) {
-		// try global package
-		name = global_data_directory + _("/") + dep.package;
-		if (!wxFileExists(name) && !wxDirExists(name)) {
-			if (report_errors)
-				handle_warning(_ERROR_1_("package not found", dep.package),false);
-			return false;
+	// mse package?
+	if (dep.package == mse_package) {
+		if (app_version < dep.version) {
+			handle_warning(_ERROR_3_("package out of date", _("Magic Set Editor"), app_version.toString(), dep.version.toString()),false);
 		}
+		return true;
+	}
+	// does the package exist?
+	if (!local.exists(dep.package) && !global.exists(dep.package)) {
+		if (report_errors)
+			handle_warning(_ERROR_1_("package not found", dep.package),false);
+		return false;
 	}
 	PackagedP package = openAny(dep.package, true);
 	if (package->version < dep.version) {
@@ -141,7 +136,290 @@ bool PackageManager::checkDependency(const PackageDependency& dep, bool report_e
 	}
 	return true;
 }
-
-void PackageManager::destroy() {
-	loaded_packages.clear();
+bool PackageManager::installedVersion(const String& package_name, Version& version_out) {
+	if (package_name == mse_package) {
+		version_out = app_version;
+		return true;
+	} else {
+		if (!local.exists(package_name) && !global.exists(package_name)) return false;
+		PackagedP package = openAny(package_name, true);
+		version_out = package->version;
+		return true;
+	}
+}
+
+void PackageManager::installedPackages(vector<InstallablePackageP>& packages) {
+	// from directories
+	vector<InstallablePackageP> more_packages;
+	global.installedPackages(packages);
+	local.installedPackages(more_packages);
+	merge(packages, more_packages);
+	// the magic appliation package
+	packages.push_back(mse_installable_package());
+}
+
+void PackageManager::install(const InstallablePackage& package) {
+	bool install_local = package.action & PACKAGE_LOCAL;
+	(install_local ? local : global).install(package);
+}
+
+// ----------------------------------------------------------------------------- : PackageDirectory
+
+void PackageDirectory::init(bool local) {
+	is_local = local;
+	if (local) {
+		init(wxStandardPaths::Get().GetUserDataDir() + _("/data"));
+	} else {
+		// determine data directory
+		String dir = wxStandardPaths::Get().GetDataDir();
+		// check if this is the actual data directory, especially during debugging,
+		// the data may be higher up:
+		//  exe path  = mse/build/debug/mse.exe
+		//  data path = mse/data
+		while (!wxDirExists(dir + _("/data"))) {
+			String d = dir;
+			dir = wxPathOnly(dir);
+			if (d == dir) {
+				// we are at the root -> 'data' not found anywhere in the path -> fatal error
+				throw Error(_("The global MSE data files can not be found, there should be a directory called 'data' with these files. The expected place to find it in was ") + wxStandardPaths::Get().GetDataDir());
+			}
+		}
+		init(dir + _("/data"));
+	}
+}
+void PackageDirectory::init(const String& dir) {
+	directory = dir;
+}
+
+String PackageDirectory::name(const String& name) const {
+	return directory + _("/") + name;
+}
+bool PackageDirectory::exists(const String& filename) const {
+	String fn = name(filename);
+	return wxFileExists(fn) || wxDirExists(fn);
+}
+
+String PackageDirectory::findFirstMatching(const String& pattern) const {
+	if (!wxDirExists(directory)) return String();
+	return wxFindFirstFile(directory + _("/") + pattern, 0);
+}
+
+bool compare_name(const PackageVersionP& a, const PackageVersionP& b) {
+	return a->name < b->name;
+}
+
+void PackageDirectory::installedPackages(vector<InstallablePackageP>& packages_out) {
+	loadDatabase();
+	// find all package files
+	vector<String> in_dir;
+	for (String s = findFirstMatching(_("*.mse-*")) ; !s.empty() ; s = wxFindNextFile()) {
+		size_t pos = s.find_last_of(_("/\\"));
+		if (pos != String::npos) s = s.substr(pos+1);
+		// TODO : check for valid package names
+		in_dir.push_back(s);
+	}
+	sort(in_dir.begin(), in_dir.end());
+	// merge with package database
+	bool db_changed = false;
+	vector<PackageVersionP>::const_iterator it1 = packages.begin();
+	vector<String>::const_iterator          it2 = in_dir.begin();
+	while (it2 != in_dir.end()) {
+		if (it1 == packages.end() || (*it1)->name > *it2) {
+			// add new package to db
+			try {
+				PackagedP pack = ::packages.openAny(*it2, true);
+				db_changed = true;
+				PackageVersionP ver(new PackageVersion(
+					is_local ? PackageVersion::STATUS_LOCAL : PackageVersion::STATUS_GLOBAL));
+				ver->check_status(*pack);
+				packages_out.push_back(new_intrusive2<InstallablePackage>(ver, new_intrusive1<PackageDescription>(*pack)));
+			} catch (const Error&) {}
+			++it2;
+		} else if ((*it1)->name < *it2) {
+			// delete package from db
+			db_changed = true;
+			++it1;
+		} else {
+			// ok, a package already in the db
+			try {
+				PackagedP pack = ::packages.openAny(*it2, true);
+				(*it1)->check_status(*pack);
+				packages_out.push_back(new_intrusive2<InstallablePackage>(*it1, new_intrusive1<PackageDescription>(*pack)));
+			} catch (const Error&) { db_changed = true; }
+			++it1, ++it2;
+		}
+	}
+	if (it1 != packages.end()) db_changed = true;
+	// has the database of installed packages changed?
+	if (db_changed) {
+		packages.clear();
+		FOR_EACH(p, packages_out) {
+			if (p->installed) packages.push_back(p->installed);
+		}
+		saveDatabase();
+	}
+}
+
+IMPLEMENT_REFLECTION(PackageDirectory) {
+	REFLECT(packages);
+}
+
+void PackageDirectory::loadDatabase() {
+	if (!packages.empty()) return;
+	String filename = databaseFile();
+	if (wxFileExists(filename)) {
+		// packages file not existing is not an error
+		shared_ptr<wxFileInputStream> file = new_shared1<wxFileInputStream>(filename);
+		if (!file->Ok()) return; // failure is not an error
+		Reader reader(file, nullptr, filename);
+		reader.handle_greedy(*this);
+		sort(packages.begin(), packages.end(), compare_name);
+	}
+}
+
+void PackageDirectory::saveDatabase() {
+	Writer writer(new_shared1<wxFileOutputStream>(databaseFile()));
+	writer.handle(*this);
+}
+String PackageDirectory::databaseFile() {
+	return name(_("packages"));
+}
+
+// ----------------------------------------------------------------------------- : PackageDirectory : installing
+
+bool PackageDirectory::install(const InstallablePackage& package) {
+	String n = name(package.description->name);
+	if (package.action & PACKAGE_REMOVE) {
+		remove_file_or_dir(n);
+	} else if ((package.action & PACKAGE_UPGRADE) || (package.action & PACKAGE_INSTALL)) {
+		remove_file_or_dir(n + _(".new"));
+		bool ok = actual_install(package, n + _(".new"));
+		if (!ok) return false;
+		move_ignored_files(n, n + _(".new"));
+		remove_file_or_dir(n);
+		rename_file_or_dir(n + _(".new"), n);
+	}
+	return true;
+}
+
+bool PackageDirectory::actual_install(const InstallablePackage& package, const String& install_dir) {
+	String name = package.description->name;
+	if (!package.installer->installer) {
+		handle_warning(_("Installer not found for package: ") + name);
+		return false;
+	}
+	Installer& installer = *package.installer->installer;
+	// install files
+	const Packaged::FileInfos& file_infos = installer.getFileInfos();
+	for (Packaged::FileInfos::const_iterator it = file_infos.begin() ; it != file_infos.end() ; ++it) {
+		String file = it->first;
+		if (!is_substr(file,0,name)) continue; // not the right package
+		// correct filename
+		file = install_dir + file.substr(name.length());
+		create_parent_dirs(file);
+		// copy file
+		InputStreamP is = installer.openIn(file);
+		wxFileOutputStream os (install_dir + _("/") + file);
+		if (!os.IsOk()) {
+			int act = wxMessageBox(_ERROR_1_("cannot create file", file), _TITLE_("cannot create file"), wxICON_ERROR | wxYES_NO);
+			if (act == wxNO) return false;
+		}
+		os.Write(*is);
+	}
+	// update package database
+	return true;
+}
+
+// ----------------------------------------------------------------------------- : PackageVersion
+
+template <> void Writer::handle(const PackageVersion::FileInfo& f) {
+	if (f.status == PackageVersion::FILE_DELETED) {
+		handle(_("D ") + f.file);
+	} else {
+		handle(format_string(_("%s%s %s"),
+			  f.status == PackageVersion::FILE_ADDED    ? _("A")
+			: f.status == PackageVersion::FILE_MODIFIED ? _("M") : _(""),
+			f.time.Format(_("%Y%m%dT%H%M%S")),
+			f.file));
+	}
+}
+template <> void Reader::handle(PackageVersion::FileInfo& f) {
+	String s; handle(s);
+	// read status
+	if (s.size() < 2) {f.status = PackageVersion::FILE_ADDED; return; }
+	f.status = s.GetChar(0) == _('M') ? PackageVersion::FILE_MODIFIED
+	         : s.GetChar(0) == _('A') ? PackageVersion::FILE_ADDED
+	         : s.GetChar(0) == _('D') ? PackageVersion::FILE_DELETED
+	         :                          PackageVersion::FILE_UNCHANGED;
+	if (f.status == PackageVersion::FILE_DELETED) {
+		if (s.GetChar(1) != _(' ')) {f.status = PackageVersion::FILE_ADDED; return; }
+		f.file = s.substr(2);
+		return;
+	} else if (f.status != PackageVersion::FILE_UNCHANGED) {
+		s = s.substr(1);
+	}
+	if (s.size() < 8+1+6+1)         {f.status = PackageVersion::FILE_ADDED; return; }
+	if (s.GetChar(8+1+6) != _(' ')) {f.status = PackageVersion::FILE_ADDED; return; } // invalid format
+	// read time, filename
+	f.time.ParseFormat(s, _("%Y%m%dT%H%M%S"));
+	f.file = s.substr(8+1+6+1);
+}
+IMPLEMENT_REFLECTION_NO_SCRIPT(PackageVersion) {
+	REFLECT_NO_SCRIPT(name);
+	REFLECT_NO_SCRIPT(version);
+	REFLECT_NO_SCRIPT(status);
+	REFLECT_NO_SCRIPT(files);
+}
+
+void PackageVersion::check_status(Packaged& package) {
+	status &= ~STATUS_MODIFIED;
+	if (!(status & STATUS_BLESSED)) status |= STATUS_MODIFIED;
+	name    = package.relativeFilename();
+	version = package.version;
+	// Merge our files list with the list from the package
+	vector<FileInfo> new_files;
+	Package::FileInfos fis = package.getFileInfos();
+	Package::FileInfos::const_iterator it1 = fis.begin();
+	vector<FileInfo>::iterator it2 = files.begin();
+//%	size_t it2 = 0, size = files.size();
+//%	bool need_sort = false;
+	while(it1 != fis.end() || it2 != files.end()) {
+		if (it1 != fis.end() && it2 != files.end() && it1->first == it2->file) {
+			DateTime mtime = package.modificationTime(*it1);
+			if (mtime != it2->time) {
+				it2->time   = mtime;
+				it2->status = FILE_MODIFIED;
+				new_files.push_back(*it2);
+				status |= STATUS_MODIFIED;
+			}
+			++it1; ++it2;
+		} else if (it1 != fis.end() && (it2 == files.end() || it1->first < it2->file)) {
+			// this is a new file
+			DateTime mtime = package.modificationTime(*it1);
+			new_files.push_back(FileInfo(it1->first, mtime, FILE_ADDED));
+			status |= STATUS_MODIFIED;
+			++it1;
+		} else {
+			// this file is no longer in the package, it was deleted
+			if (it2->status != FILE_ADDED) {
+				it2->status = FILE_DELETED;
+				new_files.push_back(*it2);
+				status |= STATUS_MODIFIED;
+			}
+			++it2;
+		}
+	}
+	swap(files,new_files);
+}
+
+inline bool is_deleted(PackageVersion::FileInfo f) {
+	return f.status == PackageVersion::FILE_DELETED;
+}
+void PackageVersion::bless() {
+	files.erase(remove_if(files.begin(),files.end(),is_deleted),files.end());
+	FOR_EACH(f,files) {
+		f.status = FILE_UNCHANGED;
+	}
+	status &= ~STATUS_MODIFIED;
+	status |= STATUS_BLESSED;
 }

src/util/io/package_manager.hpp

@@ -14,11 +14,86 @@
 #include <wx/filename.h>
 
 DECLARE_POINTER_TYPE(Packaged);
+DECLARE_POINTER_TYPE(PackageVersion);
+DECLARE_POINTER_TYPE(InstallablePackage);
 class PackageDependency;
 
+// ----------------------------------------------------------------------------- : PackageVersion
+
+/*
+
+/// Information on a package in a repository
+class PackageVersionData : public IntrusivePtrVirtualBase {
+  public:
+	PackageVersionData() {}
+	
+	String  name;						///< Name of the package
+	String  short_name;					///< Name to show on package list.
+	String  description;				///< html description
+	bool    hidden;						///< Not shown in the package list (installed automatically as a dependency)
+	
+	PackageVersionP base_version;		///< The locally installed version (if installed)
+	PackageVersionP local_version;		///< Modifications made to the locally installed version?
+	PackageVersionP remote_version;		///< The version available from the server (if available)
+	bool global;						///< The installed package is in the global location, not the user-local one
+	bool new_global;					///< 
+	
+	bool source_local;					///< Is the source
+	String source;						///< Where can the package be downloaded/found?
+	
+	DECLARE_REFLECTION();
+};
+
+class UpdateData {
+	vector<PackageDependencyP> packages;         ///< package/latest-version-number pairs
+	String                     new_updates_url;  ///< updates url has changed? 
+};
+
+IMPLEMENT_REFLECTION_NO_SCRIPT(UpdateData) {
+	REFLECT_NO_SCRIPT(packages);
+	REFLECT_NO_SCRIPT(new_updates_url);
+}
+*/
+
+// ----------------------------------------------------------------------------- : PackageDirectory
+
+/// A directory for packages
+class PackageDirectory {
+  public:
+	void init(bool local);
+	void init(const String& dir);
+	
+	/// Name of a package in this directory
+	String name(const String& name) const;
+	/// Does a package with the given name exist?
+	bool exists(const String& name) const;
+	
+	/// Find all packages that match a filename pattern (using wxFindFirst)
+	String findFirstMatching(const String& pattern) const;
+	
+	/// Get all installed packages
+	void installedPackages(vector<InstallablePackageP>& packages);
+	
+	/// Install/uninstall a package
+	bool install(const InstallablePackage& package);
+	
+	void loadDatabase();
+	void saveDatabase();
+  private:
+	bool   is_local;
+	String directory;
+	vector<PackageVersionP> packages; // sorted by name
+	
+	String databaseFile();
+	// Do the actual installation of a package
+	bool actual_install(const InstallablePackage& package, const String& install_dir);
+	
+	DECLARE_REFLECTION();
+};
+
 // ----------------------------------------------------------------------------- : PackageManager
 
-/// Package manager, loads data files from the default data directory
+/// Package manager, loads data files from the default data directory.
 /** The PackageManager ensures that each package is only loaded once.
  *  There is a single global instance of the PackageManager, called packages
  */
@@ -30,30 +105,20 @@ class PackageManager {
 	/** This function MUST be called before the program terminates, otherwise
 	 *  we could get into fights with pool allocators used by ScriptValues */
 	void destroy();
+	/// Empty the list of packages, they will all be reloaded
+	void reset();
+	
+	// --------------------------------------------------- : Packages in memory
 	
 	/// Open a package with the specified name (including extension)
 	template <typename T>
 	intrusive_ptr<T> open(const String& name) {
-		wxFileName loc(local_data_directory + _("/") + name);
-		loc.Normalize();
-		String filename = loc.GetFullPath();
-		if (!wxFileExists(filename) && !wxDirExists(filename)) {
-			wxFileName glob(global_data_directory + _("/") + name);
-			glob.Normalize();
-			filename = glob.GetFullPath();
-		}
-		// Is this package already loaded?
-		PackagedP& p = loaded_packages[filename];
+		PackagedP p = openAny(name);
 		intrusive_ptr<T> typedP = dynamic_pointer_cast<T>(p);
 		if (typedP) {
-			typedP->loadFully();
 			return typedP;
 		} else {
-			// not loaded, or loaded with wrong type (i.e. with just_header)
-			typedP = new_intrusive<T>();
-			typedP->open(filename);
-			p = typedP;
-			return typedP;
+			throw InternalError(format_string(_("Package %s loaded as wrong type"),name));
 		}
 	}
 	
@@ -75,25 +140,77 @@ class PackageManager {
 	 */
 	InputStreamP openFileFromPackage(Packaged*& package, const String& name);
 	
+	// --------------------------------------------------- : Packages on disk
+	
 	/// Check if the given dependency is currently installed
 	bool checkDependency(const PackageDependency& dep, bool report_errors = true);
+	/// Determine the latest version of the given package
+	/** If it is not installed, returns false */
+	bool installedVersion(const String& pkg, Version& version_out);
 	
-	/// Clear that cache of opened packages
-	/** Used by the update manager
-	 */
-	inline void clearPackageCache() { loaded_packages.clear(); }
+	/// Get all installed packages
+	void installedPackages(vector<InstallablePackageP>& packages);
 	
-	inline String getGlobalDataDir() const { return global_data_directory; }
-	inline String getLocalDataDir() const { return local_data_directory; }
+	/// Install/uninstall a package
+	void install(const InstallablePackage& package);
+	
+	// --------------------------------------------------- : Packages on a server
 	
   private:
 	map<String, PackagedP> loaded_packages;
-	String global_data_directory;
-	String local_data_directory;
+	PackageDirectory local, global;
 };
 
 /// The global PackageManager instance
 extern PackageManager packages;
 
+// ----------------------------------------------------------------------------- : PackageVersion
+
+/// Version information for an installed package
+class PackageVersion : public IntrusivePtrBase<PackageVersion> {
+  public:
+	PackageVersion() : status(0) {}
+	PackageVersion(int status) : status(status) {}
+	
+	String  name;
+	Version version;
+	enum Status
+	{	STATUS_LOCAL    = 0x01 ///< Package is installed in the local package dir
+	,	STATUS_GLOBAL   = 0x02 ///< Package is installed in the global package dir
+	,	STATUS_BLESSED  = 0x10 ///< Package is an official version, installed with the installer
+	,	STATUS_MODIFIED = 0x20 ///< Package has been modified after installing
+	,	STATUS_FIXED    = 0x40 ///< The package can not be uninstalled
+	};
+	int status;
+	
+	/// Check the status of the files in this package
+	void check_status(Packaged& package);
+	/// Set blessed status to true
+	void bless();
+	
+  public:
+	/// Status of a single file
+	enum FileStatus
+	{	FILE_UNCHANGED
+	,	FILE_MODIFIED
+	,	FILE_ADDED
+	,	FILE_DELETED
+	};
+	/// Information on files in this package
+	struct FileInfo {
+		inline FileInfo() {}
+		inline FileInfo(const String& file, const DateTime& time, FileStatus status)
+			: file(file), time(time), status(status)
+		{}
+		String     file;
+		DateTime   time;
+		FileStatus status;
+		inline bool operator < (const FileInfo& f) const { return file < f.file; }
+	};
+  private:
+	vector<FileInfo> files; // sorted by filename
+	DECLARE_REFLECTION();
+};
+
 // ----------------------------------------------------------------------------- : EOF
 #endif

src/util/io/reader.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include "reader.hpp"
 #include <util/vector2d.hpp>
 #include <util/error.hpp>
@@ -40,6 +41,12 @@ void Reader::addAlias(Version end_version, const Char* a, const Char* b) {
 	alias.end_version = end_version;
 }
 
+void Reader::handleIgnore(int end_version, const Char* a) {
+	if (file_app_version < end_version) {
+		if (enterBlock(a)) exitBlock();
+	}
+}
+
 void Reader::handleAppVersion() {
 	if (enterBlock(_("mse_version"))) {
 		handle(file_app_version);

src/util/io/reader.hpp

@@ -54,6 +54,8 @@ class Reader {
 	inline bool isComplex() const { return value.empty(); }
 	/// Add a as an alias for b, all keys a will be replaced with b, only if file_app_version < end_version
 	void addAlias(Version end_version, const Char* a, const Char* b);
+	/// Ignore old keys
+	void handleIgnore(int, const Char*);
 	
 	/// Read and check the application version
 	void handleAppVersion();

src/util/io/writer.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include "writer.hpp"
 #include <util/vector2d.hpp>
 #include <util/error.hpp>

src/util/io/writer.hpp

@@ -33,6 +33,7 @@ class Writer {
 	inline bool scripting() const { return false; }
 	inline bool isComplex() const { return false; }
 	inline void addAlias(int, const Char*, const Char*) {}
+	inline void handleIgnore(int, const Char*) {}
 	
 	/// Write the application version
 	void handleAppVersion();

src/util/prec.hpp

@@ -35,13 +35,6 @@
 #include <set>
 using namespace std;
 
-// MSE utility headers (ones unlikely to change and used everywhere)
-#include "for_each.hpp"
-#include "string.hpp"
-#include "smart_ptr.hpp"
-#include "index_map.hpp"
-#include "locale.hpp"
-
 // ----------------------------------------------------------------------------- : Wx Aliasses
 
 // Remove some of the wxUglyness
@@ -68,11 +61,22 @@ typedef unsigned int  UInt;
 #define nullptr 0
 
 /// A string standing for a filename, has different behaviour when reading/writing
-class FileName : public String {
+class FileName : public wxString {
   public:
-	FileName()                            {}
-	FileName(const String& s) : String(s) {}
+	FileName()                                {}
+	FileName(const wxString& s) : wxString(s) {}
 };
 
+// ----------------------------------------------------------------------------- : MSE Headers
+
+// MSE utility headers (ones unlikely to change and used everywhere)
+#include "for_each.hpp"
+#include "string.hpp"
+#include "smart_ptr.hpp"
+#include "index_map.hpp"
+#include "locale.hpp"
+#include "error.hpp"
+#include "reflect.hpp"
+
 // ----------------------------------------------------------------------------- : EOF
 #endif

src/util/reflect.hpp

@@ -141,6 +141,9 @@
  */
 #define REFLECT_ALIAS(version, old, new) tag.addAlias(version, _(old), _(new))
 
+/// Ignore things for backwards compatability for versions < 'version'
+#define REFLECT_IGNORE(version, old) tag.handleIgnore(version, _(old))
+
 /// Reflect a variable, ignores the variable for scripting
 #define REFLECT_NO_SCRIPT(var)          tag.handleNoScript(_(#var), var)
 /// Reflect a variable under the given name

src/util/rotation.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/rotation.hpp>
 #include <gfx/gfx.hpp>
 #include <data/font.hpp>

src/util/smart_ptr.hpp

@@ -240,5 +240,8 @@ inline shared_ptr<T> new_shared9(const A0& a0, const A1& a1, const A2& a2, const
 	
 #endif
 
+/// Pointer to 'anything'
+typedef intrusive_ptr<IntrusivePtrVirtualBase> VoidP;
+
 // ----------------------------------------------------------------------------- : EOF
 #endif

src/util/spec_sort.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/spec_sort.hpp>
 #include <util/error.hpp>
 

src/util/tagged_string.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/tagged_string.hpp>
 #include <stack>
 

src/util/version.cpp

@@ -6,6 +6,7 @@
 
 // ----------------------------------------------------------------------------- : Includes
 
+#include <util/prec.hpp>
 #include <util/version.hpp>
 #include <util/reflect.hpp>
 

src/util/window_id.hpp

@@ -243,9 +243,6 @@ enum ControlID {
 ,	ID_INSTALL
 ,	ID_UPGRADE
 ,	ID_REMOVE
-	// Don't use wxID_CANCEL because it makes the button look out of place
-,	ID_CANCEL
-,	ID_APPLY
 	// Auto replace window
 ,	ID_USE_AUTO_REPLACE
 ,	ID_ITEM_VALUE