ppstream: use the ppslist2 instead the ppslist

4 files changed, 524 insertions, 530 deletions

diff --git a/totem/plugin/Makefile.am b/totem/plugin/Makefile.am
index d150306..7b468aa 100644
--- a/totem/plugin/Makefile.am
+++ b/totem/plugin/Makefile.am
@@ -2,7 +2,10 @@ plugindir = $(PLUGINDIR)/ppstream
 uidir = $(plugindir)
 
 plugin_PYTHON = \
+	threadpool.py \
+	urllib2cache.py \
 	ppslist.py \
+	ppslist2.py \
 	ppstream.py \
 	PPSVideoList.py
 
diff --git a/totem/plugin/ppstream.py b/totem/plugin/ppstream.py
index 034b0d4..b561863 100644
--- a/totem/plugin/ppstream.py
+++ b/totem/plugin/ppstream.py
@@ -18,7 +18,8 @@ from traceback import print_exc
 
 from ppstream_translation import _
 
-import ppslist
+import threadpool
+import ppslist2 as ppslist
 import PPSVideoList
 
 try:
@@ -66,13 +67,15 @@ def unescape_xml (data):
 gconf_key = '/apps/totem/plugins/ppstream'
 pps_cache_dir = os.path.expanduser ('~/.local/share/totem/plugin/ppstream')
 
-
+pps_http_cache_dir = os.path.join (pps_cache_dir, "http")
 pps_xmls_dir = os.path.join (pps_cache_dir, "xmls")
 pps_images_dir = os.path.join (pps_cache_dir, "images")
 for p in [ pps_xmls_dir, pps_images_dir ]:
 	if not os.path.exists (p):
 		os.makedirs (p)
 
+ppslist.CachePath = pps_http_cache_dir
+
 ### special category ids
 CATEGORY_ID_FAVORITES = 0
 CATEGORY_ID_RECENT = -1
@@ -101,52 +104,6 @@ def xml_child_value (node, id):
 		return child.nodeValue
 	return None
 
-class ProcessThread (threading.Thread):
-	def __init__ (self, data, process, callback, *args, **kwargs):
-		self.data = data
-		self.process = process
-		self.callback = callback
-		self.args = args
-		self.kwargs = kwargs
-		threading.Thread.__init__ (self)
-
-	def run (self):
-		try:
-			res = self.process (self.data, *self.args, **self.kwargs)
-		except Exception, e:
-			print "Couldn't process data: ", e
-			print_exc()
-			res = None
-
-		gobject.idle_add (self.publish_results, res)
-
-	def publish_results(self, res):
-		self.callback (res, *self.args, **self.kwargs)
-		return False
-
-class DownloadThread (threading.Thread):
-	def __init__ (self, url, callback, *args, **kwargs):
-		self.url = url
-		self.callback = callback
-		self.args = args
-		self.kwargs = kwargs
-		threading.Thread.__init__ (self)
-
-	def run (self):
-		for i in range (3):
-			try:
-				res = urllib.urlopen (self.url).read ()
-			except Exception, e:
-				print "Coudn't open url: ", e
-				res = None
-			if res:
-				break
-		gobject.idle_add (self.publish_results, res)
-
-	def publish_results(self, res):
-		self.callback (res, *self.args, **self.kwargs)
-		return False
-
 class RetrieveImageTask:
 	pass
 
@@ -397,7 +354,7 @@ class PPStream (totem.Plugin):
 		self.notebook = self.builder.get_object ("pps_notebook")
 		self.notebook.connect ("switch-page", self.on_notebook_page_changed)
 
-		self.notebook_pages = ["categories", "movies", "files"]
+		self.notebook_pages = ["categories", "movies"]
 		self.current_treeview_name = "categories"
 		self.load_config ()
 		self.setup_favorites ()
@@ -405,7 +362,6 @@ class PPStream (totem.Plugin):
 		self.setup_search ()
 		self.setup_categories ()
 		self.setup_movies ()
-		self.setup_files ()
 		self.setup_page_ui ()
 		self.setup_config_dialog ()
 
@@ -416,6 +372,7 @@ class PPStream (totem.Plugin):
 
 		totem_object.add_sidebar_page ("ppstream", _("PPStream"), self.vbox)
 
+		self.threadpool = threadpool.ThreadPool(8)
 		self.retrieveimage = RetrieveImage ()
 		self.retrieveimage.start ()
 
@@ -433,6 +390,7 @@ class PPStream (totem.Plugin):
 		self.movies_count += 1
 		self.classes = []
 
+		self.threadpool.wait_completion ()
 		self.retrieveimage.cancel ()
 		self.retrieveimage.join ()
 
@@ -453,6 +411,37 @@ class PPStream (totem.Plugin):
 		button.set_active (self.configs['compatible'])
 		return dialog
 
+	def add_task(self, task, *args, **kwargs):
+		self.threadpool.add_task(task, *args, **kwargs)
+
+	def add_process_task(self, data, process, callback, *args, **kwargs):
+		class ProcessTask:
+			def __init__(self, data, proc, callback, *args, **kwargs):
+				self.data = data
+				self.process = proc
+				self.callback = callback
+				self.args = args
+				self.kwargs = kwargs
+
+			def run(self):
+				try:
+					res = self.process (self.data, *self.args, **self.kwargs)
+				except Exception, e:
+					print "Couldn't process data: ", e
+					print_exc()
+					res = None
+
+				gobject.idle_add (self.publish_results, res)
+
+			def publish_results(self, res):
+				self.callback (res, *self.args, **self.kwargs)
+				return False
+
+		task = ProcessTask(data, process, callback, *args, **kwargs)
+		def proxy(task):
+			task.run()
+		self.add_task(proxy, task)
+
 	def start_idle_task (self):
 		if self.configs['compatible']:
 			self.source_id = gobject.idle_add (self.on_idle)
@@ -492,9 +481,6 @@ class PPStream (totem.Plugin):
 		if force:
 			self.ppslist.reset()
 
-		if self.ppslist.getClasses() == 0:
-			return
-
 		self.movie_class_path = ()
 		self.subclass_index = 0
 		self.categories_count += 1
@@ -507,25 +493,78 @@ class PPStream (totem.Plugin):
 		self.load_special_classes ()
 
 		### download classes from pps.tv
-		thread = ProcessThread (None,
-					self.__fetch_classes,
-					self.on_classes_fetched,
-					self.categories_count)
-		thread.start()
+		self.add_process_task(None,
+				      self.__fetch_classes,
+				      self.on_classes_fetched,
+				      self.categories_count)
 
 		gobject.timeout_add (300,
 				     self.update_categories_progress,
 				     self.categories_count)
 
 	def on_classes_fetched (self, res, count):
-		self.fetching_classes = False
 		if not res:
-			print "Couldn't load ppstream playlist"
+			print "Couldn't download ppstream playlist"
+			return
+
+		### check if this is a canceled task
+		if self.categories_count != count:
+			return
+
+		### parse the class list
+		self.parse_class (res)
+
+	def parse_class (self, res):
+		### parse the class list
+		classes = self.ppslist.parseClasses(res)
+		for cls in classes:
+			cls.fetching_movies = False
+		self.ppslist.updateClasses(classes)
+
+		### fetch subclasses
+		self.fetch_subclasses()
+
+	def __fetch_subclasses (self, *args):
+		self.fetching_classes = True
+		classes = self.ppslist.getClasses()
+		curcls = classes[self.subclass_index]
+		return self.ppslist.fetchSubclasses(curcls)
+
+	def fetch_subclasses (self):
+		### download classes from pps.tv
+		self.add_process_task (None,
+				       self.__fetch_subclasses,
+				       self.on_subclasses_fetched,
+				       self.categories_count)
+
+	def on_subclasses_fetched (self, res, count):
+		if not res:
+			print "Couldn't download ppstream playlist"
 			return
 
 		### check if this is a canceled task
-		if self.categories_count == count:
-			self.parse_class (res)
+		if self.categories_count != count:
+			return
+
+		### parse the subclass list
+		self.parse_subclass (res)
+		self.subclass_index += 1
+		if self.subclass_index == len(self.ppslist.getClasses()):
+			self.fetching_classes = False
+		else:
+			self.fetch_subclasses()
+
+	def parse_subclass (self, res):
+		### parse the class list
+		classes = self.ppslist.getClasses()
+		curcls = classes[self.subclass_index]
+		subclasses = self.ppslist.parseSubclasses(curcls, res)
+		for cls in subclasses:
+			cls.fetching_movies = False
+		self.ppslist.updateSubclasses(curcls, subclasses)
+
+		### fill the treestore
+		self.fill_categories (curcls.id)
 
 	def __parse_movies (self, res, class_path, page, count):
 		cls = self.get_movie_class (class_path)
@@ -551,19 +590,11 @@ class PPStream (totem.Plugin):
 			cls.fetching_movies = False
 			return
 
-		### If the size of content is larger then 4k, parse it in another thread.
-		### Blocking the main thread is bad.
-		if res and len(res) > 1024 * 2:
-			cls.parsing_movies = True
-			process = ProcessThread (res, self.__parse_movies,
-						 self.add_parsed_movies,
-						 class_path, page, count)
-			process.start ()
-			self.update_progress_bar ()
-			return
-
-		result = self.__parse_movies (res, class_path, page, count)
-		self.add_parsed_movies (result, class_path, page, count)
+		cls.parsing_movies = True
+		self.add_process_task (res, self.__parse_movies,
+				       self.add_parsed_movies,
+				       class_path, page, count)
+		self.update_progress_bar ()
 
 	def add_parsed_movies (self, result, class_path, page, count):
 		if self.movies_count != count or page != self.movies_page:
@@ -575,23 +606,13 @@ class PPStream (totem.Plugin):
 
 		cls.fetching_movies = False
 
+		print 'updating movie list'
 		if result:
-			## for movie in result:
-			## 	print movie
+			for movie in result:
+				movie.fetching_meta_data = False
 			self.fill_movies (result, class_path)
 		self.set_page_max(cls.getMaxPage())
-
-	def parse_class (self, res):
-		### parse the class list
-		classes = self.ppslist.parseClasses(res)
-		for cls in classes:
-			cls.fetching_movies = False
-		self.ppslist.updateClasses(classes)
-
-		### fill category list
-		self.clear_pages ('categories')
-		for cls in self.ppslist.getClasses():
-			self.fill_categories (cls.id)
+		print 'movie list parsed'
 
 	def get_movie_class (self, class_path):
 		def find_class (classes, clsid):
@@ -602,14 +623,16 @@ class PPStream (totem.Plugin):
 		if not class_path:
 			return None
 		classes = self.ppslist.getClasses()
-		return find_class (classes, class_path[0])
+		cls = find_class (classes, class_path[0])
+		if cls and len(class_path) > 1:
+			return find_class (cls.children, class_path[1])
+		return cls
 
 	def download_movies (self, callback, class_path, page, count):
 		cls = self.get_movie_class (class_path)
-		thread = ProcessThread (None, self.__fetch_movies,
-					callback, class_path,
-					page, count)
-		thread.start()
+		self.add_process_task (None, self.__fetch_movies,
+				       callback, class_path,
+				       page, count)
 
 	def __fetch_movies(self, dummy, class_path, page, *args):
 		cls = self.get_movie_class (class_path)
@@ -624,18 +647,84 @@ class PPStream (totem.Plugin):
 		self.download_movies (self.on_movies_fetched, class_path,
 				     self.movies_page, self.movies_count)
 
+	def update_movie_meta_data(self, class_path, movie):
+		print 'updating movie meta data:', movie
+		cls = self.get_movie_class (class_path)
+		treeview_name = 'movies'
+		model = self.liststore[treeview_name]
+		it = model.get_iter(movie.model_path)
+		tip = self.get_movie_tooltip(movie)
+		model.set (it, 3, tip)
+		self.fetch_movie_poster(class_path, movie, it)
+
+	def parse_movie_meta_data(self, res, class_path, movie, count):
+		print 'parsing movie meta data:', movie
+		cls = self.get_movie_class (class_path)
+		if cls.desc_url:
+			cls.parseMetaData(res)
+			movie.updateMetaData(cls.meta_data)
+		else:
+			movie.parseBKMetaData(res)
+		#print 'meta data:', movie.meta_data
+
+	def on_movie_meta_data_parsed(self, dummy, class_path, movie, count):
+		if count != self.movies_count:
+			return
+		movie.fetching_meta_data = False
+		self.update_movie_meta_data(class_path, movie)
+
+	def on_movie_meta_data_fetched (self, res, class_path, movie, count,
+					new = True):
+		if self.movies_count != count:
+			return
+
+		### check if the server repeats the last page
+		cls = self.get_movie_class (class_path)
+
+		if res is None:
+			movie.fetching_meta_data = False
+			return
+
+		self.add_process_task (res, self.parse_movie_meta_data,
+				       self.on_movie_meta_data_parsed,
+				       class_path, movie, count)
+
+	def __fetch_movie_meta_data(self, dummy, class_path, movie, *args):
+		cls = self.get_movie_class (class_path)
+		if cls.desc_url:
+			return cls.fetchMetaData()
+		return movie.fetchBKMetaData()
+
+	def fetch_movie_meta_data(self, class_path, movie):
+		cls = self.get_movie_class (class_path)
+		if movie.meta_data:
+			return
+
+		print 'fetching movie meta data:', movie
+		movie.fetching_meta_data = True
+		self.add_process_task (None, self.__fetch_movie_meta_data,
+				       self.on_movie_meta_data_fetched,
+				       class_path, movie, self.movies_count)
+
 	def on_category_row_activated(self, treeview, path, view_column,
 				      data = None):
 		model, rows = treeview.get_selection ().get_selected_rows ()
 		iter = model.get_iter (rows[0])
 		title = model.get_value (iter, 3)
 		classid = model.get_value (iter, 0)
+		subclassid = model.get_value (iter, 2)
+
+		if subclassid:
+			class_path = (classid, subclassid)
+		else:
+			class_path = (classid,)
 
-		class_path = (classid,)
+		cls = self.get_movie_class(class_path)
+		if cls.children and subclassid == 0:
+			return
 		#print title, class_path
 
 		self.movies_page = 0
-		self.movies_page = 0
 		self.show_movies (class_path)
 		self.notebook.set_current_page (1)
 
@@ -766,6 +855,7 @@ class PPStream (totem.Plugin):
 		movie = copy.copy(movie)
 		movie.model_path = ()
 		self.favorites.addMovies([movie])
+		self.favorites.setCount(len(self.favorites.getMovies()))
 
 		if not update:
 			return
@@ -781,6 +871,7 @@ class PPStream (totem.Plugin):
 			print "Couldn't find ", movie, ' in favorites'
 			return
 		movies.remove(movie)
+		self.favorites.setCount(len(self.favorites.getMovies()))
 
 		if not update:
 			return
@@ -868,6 +959,7 @@ class PPStream (totem.Plugin):
 		self.recent.resetMovies()
 		movie = copy.copy(movie)
 		self.recent.setMovies([movie] + movies)
+		self.recent.setCount(len(self.recent.getMovies()))
 
 		if self.movie_class_path == (CATEGORY_ID_RECENT,):
 			if old_index < 0:
@@ -908,22 +1000,32 @@ class PPStream (totem.Plugin):
 
 	def fill_all_categories (self):
 		treeview_name = 'categories'
-		treeview = self.liststore[treeview_name]
+		model = self.liststore[treeview_name]
 		self.clear_pages (treeview_name)
 		for cls in self.ppslist.getClasses():
-			iter = treeview.append(None)
-			s = '%d/%d' % (len(cls.getMovies()), cls.getMaxPage())
-			treeview.set(iter,
-				     0, cls.id,
-				     1, 0,
-				     2, 0,
-				     3, cls.title,
-				     4, s)
-			cls.model_path = treeview.get_path(iter)
-
+			iter = model.append(None)
+			s = '%d' % cls.count
+			model.set(iter,
+				  0, cls.id,
+				  1, 0,
+				  2, 0,
+				  3, cls.title,
+				  4, s)
+			cls.model_path = model.get_path(iter)
+
+			for subcls in cls.children:
+				child_iter = model.append(iter)
+				s = '%d' % subcls.count
+				model.set(child_iter,
+					  0, cls.id,
+					  1, 0,
+					  2, subcls.id,
+					  3, subcls.title,
+					  4, s)
+				subcls.model_path = model.get_path(child_iter)
 	def fill_categories (self, clsid):
 		treeview_name = 'categories'
-		treeview = self.liststore[treeview_name]
+		model = self.liststore[treeview_name]
 
 		cls = None
 		for cls in self.ppslist.getClasses():
@@ -932,22 +1034,32 @@ class PPStream (totem.Plugin):
 		if not cls:
 			return
 
-		s = '%d/%d' % (len(cls.getMovies()), cls.getMaxPage())
-		iter = treeview.append(None)
-		treeview.set(iter,
-			     0, cls.id,
-			     1, 0,
-			     2, 0,
-			     3, cls.title,
-			     4, s)
-		cls.model_path = treeview.get_path(iter)
+		s = '%d' % cls.count
+		iter = model.append(None)
+		model.set(iter,
+			  0, cls.id,
+			  1, 0,
+			  2, 0,
+			  3, cls.title,
+			  4, s)
+		cls.model_path = model.get_path(iter)
+		for subcls in cls.children:
+			child_iter = model.append(iter)
+			s = '%d' % subcls.count
+			model.set(child_iter,
+				  0, cls.id,
+				  1, 0,
+				  2, subcls.id,
+				  3, subcls.title,
+				  4, s)
+			subcls.model_path = model.get_path(child_iter)
 		#print cls.id, cls.title
 
 	def update_category (self, cls):
 		treeview_name = 'categories'
 		liststore = self.liststore[treeview_name]
 		iter = liststore.get_iter (cls.model_path)
-		s = '%d/%d' % (len(cls.getMovies()), cls.getMaxPage())
+		s = '%d' % cls.count
 		liststore.set (iter, 4, s)
 
 	def update_current_category (self):
@@ -971,7 +1083,6 @@ class PPStream (totem.Plugin):
 		if movie.length:
 			tip += '\n' + _("Length: ") + str(movie.length) + _('Min')
 		tip += '\n' + _("Score: ") + str(movie.score)
-		tip += '\n' + _("Episodes: ") + str(movie.cn)
 		tip += '\n' + _("Description: ") + '\n' + movie.desc
 		if hasattr(file, 'longdesc'):
 			tip += '\n' + _("Long Description: ") + movie.longdesc
@@ -995,6 +1106,7 @@ class PPStream (totem.Plugin):
 			pass
 
 	def recode_mrl (self, url):
+		return url
 		if not self.can_recode_mrl:
 			return url
 		path, qs = urllib.splitquery (url)
@@ -1009,126 +1121,40 @@ class PPStream (totem.Plugin):
 			qs = ''
 		return path + qs
 
-	def storelist_append_movie (self, treeview, movie, class_path, pos = -1):
-		files = movie.getFiles(self.files_page)
-		#if not movie.getFiles():
-		#	return
-		f = None
-		if len(files):
-			f = files[0]
-			origurl = f.url
-		else:
-			origurl = ''
+	def fetch_movie_poster(self, class_path, movie, it):
+		print 'fetching movie posters:', movie, movie.image
+		if not self.configs['show_posters']:
+			return
+		if movie.pixbuf or not movie.image:
+			return
+
+		print 'fetching movie poster image:', movie, movie.image
+		self.retrieveimage.retrieve (movie.image,
+					     self.on_image_retrieved,
+					     movie, it, class_path,
+					     self.movies_count)
+
+	def append_movie (self, model, movie, class_path, pos = -1):
+		origurl = movie.url
 		tip = self.get_movie_tooltip (movie)
 		pixbuf = movie.pixbuf
 		if not self.configs['show_posters']:
 			pixbuf = None
 		url = self.recode_mrl (origurl)
 		if pos == -1:
-			it = treeview.append ([pixbuf, movie.title, url, tip,
-					       repr(class_path), origurl])
-		else:
-			it = treeview.insert (0)
-			treeview.set (it,
-				      0, pixbuf,
-				      1, movie.title,
-				      2, url,
-				      3, tip,
-				      4, repr(class_path),
-				      5, f.url)
-		if not movie.pixbuf and movie.smallimage and self.configs['show_posters']:
-			self.retrieveimage.retrieve (movie.smallimage, self.on_image_retrieved,
-						     movie, it, class_path,
-						     self.movies_count)
-		movie.model_path = treeview.get_path(it)
-
-	def show_files_current_page(self):
-		movie = self.current_movie
-		if self.current_treeview_name != 'files':
-			return
-
-		if movie:
-			self.set_page_max(movie.getMaxPage())
-			self.set_current_page(self.files_page + 1)
-		else:
-			self.set_page_max(1)
-			self.set_current_page(1)
-
-	def fill_files (self, movie):
-		treeview_name = 'files'
-		self.clear_pages (treeview_name)
-		treeview = self.liststore[treeview_name]
-		files = movie.getFiles(self.files_page)
-		num_files = len(files)
-		nr = 0
-		for f in files:
-			no = str(nr + 1) + '/' + str(num_files)
-			it = treeview.append ([no, f.title, self.recode_mrl (f.url), f.url])
-			nr += 1
-		self.show_files_current_page()
-
-	def show_files (self, movie):
-		if self.current_movie and self.current_movie != movie:
-			self.files_page = 0
-		self.current_movie = movie
-		if movie.getFiles(self.files_page):
-			self.fill_files(movie)
+			it = model.append ([pixbuf, movie.title, url, tip,
+					    repr(class_path), origurl])
 		else:
-			self.clear_pages('files')
-			self.fetch_files(movie, self.files_page)
-			self.show_files_current_page()
-
-	def download_files (self, callback, movie, page, show, count):
-		thread = ProcessThread (None, self.__fetch_files,
-					callback, movie, page, show, count)
-		thread.start()
-
-	def __fetch_files(self, dummy, movie, page, show, count):
-		cls = movie.parent
-		assert (cls is not None)
-		#print movie, page, movie.baseurl
-		if not movie.baseurl:
-			return self.ppslist.fetchMovie(cls, movie)
-		return self.ppslist.fetchMoviePlayList(cls, movie, page)
-
-	def fetch_files (self, movie, page = 0, show = True):
-		self.download_files (self.on_files_fetched, movie,
-				     page, show, self.movies_count)
-
-	def on_files_fetched(self, res, movie, page, show, movies_count):
-		curpage = page
-		if res:
-			if movie.baseurl:
-				result = self.ppslist.parseMoviePlayList(movie.parent,
-									movie, res,
-									page)
-				if not result:
-					files = []
-				else:
-					files = result[0]
-					curpage = result[2]
-			else:
-				files = self.ppslist.parseMovie(movie.parent, movie, res)
-			if not files and not movie.baseurl:
-				url = self.ppslist.parseMoviePlayListUrl(movie.parent,
-									 movie, res)
-				assert(curpage == 0)
-				### the default page
-				curpage = -1
-				self.download_files(self.on_files_fetched, movie,
-						    curpage, show, movies_count)
-			else:
-				if curpage != page and self.movies_count == movies_count and \
-				   movie == self.current_movie:
-					#print 'updating files_page to', curpage
-					self.files_page = curpage
-
-				movie.setFiles(files, curpage)
-		if show:
-			if self.movies_count == movies_count and \
-			   movie == self.current_movie and curpage == self.files_page:
-				self.fill_files(movie)
-		self.movies_update_files_status()
+			it = model.insert (0)
+			model.set (it,
+				   0, pixbuf,
+				   1, movie.title,
+				   2, url,
+				   3, tip,
+				   4, repr(class_path),
+				   5, origurl)
+		movie.model_path = model.get_path(it)
+		self.fetch_movie_meta_data(class_path, movie)
 
 	def fill_all_movies (self, class_path):
 		cls = self.get_movie_class (class_path)
@@ -1138,7 +1164,7 @@ class PPStream (totem.Plugin):
 		treeview_name = 'movies'
 		liststore = self.liststore[treeview_name]
 		for movie in cls.getPage(self.movies_page):
-			self.storelist_append_movie (liststore, movie, class_path)
+			self.append_movie (liststore, movie, class_path)
 
 		self.update_category (cls)
 
@@ -1148,7 +1174,7 @@ class PPStream (totem.Plugin):
 		treeview_name = 'movies'
 		liststore = self.liststore[treeview_name]
 		for movie in movies:
-			self.storelist_append_movie (liststore, movie, class_path)
+			self.append_movie (liststore, movie, class_path)
 
 		self.update_current_category ()
 
@@ -1179,7 +1205,7 @@ class PPStream (totem.Plugin):
 		if not self.configs['show_posters']:
 			self.retrieveimage.cancel_tasks ()
 
-		self.clear_pages (['movies', 'files'])
+		self.clear_pages (['movies'])
 
 		cls = self.get_movie_class (class_path)
 
@@ -1252,174 +1278,13 @@ class PPStream (totem.Plugin):
 		selection = treeview.get_selection ()
 		selection.connect ("changed", self.on_movies_selection_changed)
 
-	def on_add_to_playlist (self, treeview):
-		treeview_name = 'files'
-		treeview = self.treeview[treeview_name]
-		selection = treeview.get_selection ()
-		model = treeview.get_model ()
-		model, rows = selection.get_selected_rows ()
-
-		for row in rows:
-			iter = model.get_iter (row)
-			mrl = model.get_value (iter, 2)
-			self.totem.action_remote (totem.REMOTE_COMMAND_ENQUEUE, mrl)
-
-		### add selected movie to recent list
-		treeview_name = 'movies'
-		treeview = self.treeview[treeview_name]
-		selection = treeview.get_selection ()
-		model = treeview.get_model ()
-		model, rows = selection.get_selected_rows ()
-		self.add_movie_to_recent (treeview, rows[0])
-
-	def on_copy_location (self, action):
-		treeview_name = 'files'
-		treeview = self.treeview[treeview_name]
-		selection = treeview.get_selection ()
-		model = treeview.get_model ()
-		model, rows = selection.get_selected_rows ()
-		iter = model.get_iter (rows[0])
-		mrl = model.get_value (iter, 3)
-
-		clip = gtk.Clipboard ()
-		clip.set_text (mrl)
-		clip = gtk.Clipboard (display = gtk.gdk.display_get_default(),
-				      selection = "PRIMARY")
-		clip.set_text (mrl)
-
-	def setup_files_action (self):
-		datadir = os.path.dirname (__file__)
-                filename = os.path.join (datadir, 'ppstream-ui.xml')
-
-		self.files_uimanager = self.builder.get_object ('pps-list-ui-manager')
-		self.files_uimanager.add_ui_from_file(filename)
-
-		action_group = gtk.ActionGroup(name = 'pps-list-action-group')
-		self.files_uimanager.insert_action_group(action_group, 0)
-
-		action = gtk.Action(name = 'add-to-playlist',
-				    label = _('_Add to Playlist'),
-				    tooltip = _('Add the video to the playlist'),
-				    stock_id = 'gtk-add')
-		action.connect ('activate', self.on_add_to_playlist)
-		action_group.add_action(action)
-
-		action = gtk.Action(name = 'copy-location',
-				    label = _('_Copy Location'),
-				    tooltip = _('Copy the location to the clipboard'),
-				    stock_id = 'gtk-copy')
-		action.connect ('activate', self.on_copy_location)
-		action_group.add_action(action)
-
-	def files_show_popup_menu (self, event = None):
-		treeview_name = 'files'
-		treeview = self.treeview [treeview_name]
-		selection = treeview.get_selection ()
-
-		if event:
-			button = event.button
-			time = event.time
-			path = treeview.get_path_at_pos (int(event.x), int(event.y))
-			if not path or not selection.path_is_selected (path[0]):
-				selection.unselect_all ()
-		else:
-			time = gtk.get_current_event_time ()
-			button = None
-
-		count = selection.count_selected_rows ()
-		if not count:
-			return False
-
-		action_group = self.files_uimanager.get_action_groups () [0]
-		action = action_group.get_action ("copy-location")
-		action.set_sensitive (count == 1)
-
-		menu = self.files_uimanager.get_widget ("/pps-list-popup")
-		menu.select_first (False)
-		menu.popup (None, None, None, button, time)
-
-		return True
-
-	def on_files_popup_menu (self):
-		self.files_show_popup_menu ()
-
-	def on_files_button_pressed (self, widget, event, user_data = None):
-		if event.type == gtk.gdk.BUTTON_PRESS and event.button == 3:
-			return self.files_show_popup_menu (event)
-
-		return False
-
-	def setup_files (self):
-		treeview_name = 'files'
-		treeview = self.builder.get_object ("pps_treeview_" + treeview_name)
-		renderer = gtk.CellRendererText()
-		renderer.set_property("xalign", 0.0)
-
-		column = gtk.TreeViewColumn(_("No"), renderer, text = 0)
-		column.set_clickable(True)
-		treeview.append_column(column)
-
-		column = gtk.TreeViewColumn(_("Title"), renderer, text = 1)
-		treeview.append_column(column)
-
-		column = gtk.TreeViewColumn(_("Mrl"), renderer, text = 2)
-		treeview.append_column(column)
-
-		self.vadjust[treeview_name] = treeview.get_vadjustment ()
-		self.vadjust[treeview_name].connect ("value-changed", self.on_value_changed)
-		vscroll = self.builder.get_object ("pps_scrolled_window_" + treeview_name).get_vscrollbar ()
-		vscroll.connect ("button-press-event", self.on_button_press_event)
-		vscroll.connect ("button-release-event", self.on_button_release_event)
-
-		self.liststore[treeview_name] = self.builder.get_object ("pps_liststore_" + treeview_name)
-		self.treeview[treeview_name] = treeview
-		treeview.set_model (self.liststore[treeview_name])
-
-		treeview.connect("row_activated", self.on_files_treeview_row_activated)
-
-		self.setup_files_action ()
-		treeview.connect ("popup-menu", self.on_files_popup_menu)
-		treeview.connect ("button-press-event", self.on_files_button_pressed)
-
-		selection = treeview.get_selection ()
-		selection.set_mode (gtk.SELECTION_MULTIPLE)
-
-		self.files_page = 0
-		self.files_count = 0
-
-	def on_files_treeview_row_activated(self, treeview, path, view_column, data=None):
-		model, rows = treeview.get_selection ().get_selected_rows ()
-		try:
-			iter = model.get_iter (rows[0])
-		except IndexError:
-			print 'activated invalid index:', rows[0]
-			return
-
-		### play the movie
-		title = model.get_value (iter, 1)
-		mrl = model.get_value (iter, 2)
-		#print title, mrl
-		if hasattr(self.totem, "add_to_playlist_and_play"):
-			self.totem.add_to_playlist_and_play (mrl, title, False)
-		else:
-			self.totem.action_remote (totem.REMOTE_COMMAND_REPLACE, mrl)
-
-		### add selected movie to recent list
-		movies_treeview = self.treeview['movies']
-		selection = movies_treeview.get_selection ()
-		movies_model, movies_rows = selection.get_selected_rows ()
-		self.add_movie_to_recent (movies_treeview, movies_rows[0])
-
 	def on_movies_selection_changed (self, selection):
 		model, rows = selection.get_selected_rows ()
 		if not rows:
-			self.clear_pages ('files')
 			self.current_movie = None
-			self.files_page = 0
-			self.show_files_current_page()
 			return
 
-		all_files_fetched = True
+		all_meta_data_fetched = True
 		for row in rows:
 			iter = model.get_iter(row)
 			if type (model) is gtk.TreeModelFilter:
@@ -1431,49 +1296,11 @@ class PPStream (totem.Plugin):
 
 			index = model.get_path(iter)[0]
 			movie = cls.getPage(self.movies_page)[index]
-			if not movie.getFiles(self.files_page):
-				all_files_fetched = False
-				if row != rows[0]:
-					self.fetch_files(movie, False)
-			if row == rows[0]:
-				self.show_files(movie)
-		if self.movie_class_path == (CATEGORY_ID_FAVORITES,):
-			self.add_to_favorites_button.set_sensitive(False)
-		else:
-			self.add_to_favorites_button.set_sensitive(all_files_fetched)
-
-	def movies_update_files_status(self):
-		selection = self.treeview['movies'].get_selection ()
-		model, rows = selection.get_selected_rows ()
-		if not rows:
-			self.clear_pages ('files')
-			return
-
-		all_files_fetched = True
-		for row in rows:
-			iter = model.get_iter(row)
-			if type (model) is gtk.TreeModelFilter:
-				iter = model.convert_iter_to_child_iter (iter)
-				model = model.get_model ()
+			if not movie.meta_data:
+				all_meta_data_fetched = False
+			#self.fetch_meta_data(movie)
 
-			class_path = eval(model.get_value (iter, 4))
-			cls = self.get_movie_class (class_path)
-
-			index = model.get_path(iter)[0]
-			movie = cls.getPage(self.movies_page)[index]
-			if not movie.getFiles(self.files_page):
-				all_files_fetched = False
-			else:
-				url = model.get(iter, 2)[0]
-				if not url:
-					url = movie.getFiles(self.files_page)[0].url
-					model.set(iter, 2, self.recode_mrl(url))
-					model.set(iter, 5, url)
-
-		if self.movie_class_path == (CATEGORY_FAVORITES,):
-			self.add_to_favorites_button.set_sensitive(False)
-		else:
-			self.add_to_favorites_button.set_sensitive(all_files_fetched)
+		self.add_to_favorites_button.set_sensitive(True)
 
 	def movies_goto_first_page(self):
 		cls = self.get_movie_class(self.movie_class_path)
@@ -1484,19 +1311,11 @@ class PPStream (totem.Plugin):
 
 		self.show_movies(self.movie_class_path, force = True)
 
-	def files_goto_first_page(self):
-		if self.files_page == 0:
-			return
-		self.files_page = 0
-		self.show_files(self.current_movie)
-
 	def on_first_page_clicked(self, *args):
 		if self.movie_class_path == ():
 			return
 		if self.current_treeview_name == 'movies':
 			self.movies_goto_first_page()
-		elif self.current_treeview_name == 'files':
-			self.files_goto_first_page()
 
 	def movies_goto_prev_page(self):
 		if self.movies_page < 1:
@@ -1504,19 +1323,11 @@ class PPStream (totem.Plugin):
 		self.movies_page -= 1
 		self.show_movies(self.movie_class_path, force = True)
 
-	def files_goto_prev_page(self):
-		if self.files_page < 1:
-			return
-		self.files_page -= 1
-		self.show_files(self.current_movie)
-
 	def on_prev_page_clicked(self, *args):
 		if self.movie_class_path == ():
 			return
 		if self.current_treeview_name == 'movies':
 			self.movies_goto_prev_page()
-		elif self.current_treeview_name == 'files':
-			self.files_goto_prev_page()
 
 	def movies_goto_next_page(self):
 		cls = self.get_movie_class(self.movie_class_path)
@@ -1527,19 +1338,11 @@ class PPStream (totem.Plugin):
 
 		self.show_movies(self.movie_class_path, force = True)
 
-	def files_goto_next_page(self):
-		if self.files_page >= self.current_movie.getMaxPage() - 1:
-			return
-		self.files_page += 1
-		self.show_files(self.current_movie)
-
 	def on_next_page_clicked(self, *args):
 		if self.movie_class_path == ():
 			return
 		if self.current_treeview_name == 'movies':
 			self.movies_goto_next_page()
-		elif self.current_treeview_name == 'files':
-			self.files_goto_next_page()
 
 	def movies_goto_last_page(self):
 		cls = self.get_movie_class(self.movie_class_path)
@@ -1550,19 +1353,11 @@ class PPStream (totem.Plugin):
 
 		self.show_movies(self.movie_class_path, force = True)
 
-	def files_goto_last_page(self):
-		if self.files_page == self.current_movie.getMaxPage() - 1:
-			return
-		self.files_page = self.current_movie.getMaxPage() - 1
-		self.show_files(self.current_movie)
-
 	def on_last_page_clicked(self, *args):
 		if self.movie_class_path == ():
 			return
 		if self.current_treeview_name == 'movies':
 			self.movies_goto_last_page()
-		elif self.current_treeview_name == 'files':
-			self.files_goto_last_page()
 
 	def movies_goto_page(self, page):
 		cls = self.get_movie_class(self.movie_class_path)
@@ -1573,15 +1368,6 @@ class PPStream (totem.Plugin):
 		self.movies_page = page
 		self.show_movies(self.movie_class_path, force = True)
 
-	def files_goto_page(self, page):
-		movie = self.current_movie
-		if page >= movie.getMaxPage():
-			return
-		if page == self.files_page:
-			return
-		self.files_page = page
-		self.show_files(movie)
-
 	def on_goto_page_clicked(self, *args):
 		if self.movie_class_path == ():
 			return
@@ -1589,8 +1375,6 @@ class PPStream (totem.Plugin):
 		page = self.which_page_button.get_value_as_int() - 1
 		if self.current_treeview_name == 'movies':
 			self.movies_goto_first_page()
-		elif self.current_treeview_name == 'files':
-			self.files_goto_first_page()
 
 	def set_page_max(self, max_page):
 		adjustment = self.which_page_button.get_adjustment()
@@ -1629,17 +1413,7 @@ class PPStream (totem.Plugin):
 		self.current_treeview_name = self.notebook_pages[page_num]
 		self.add_to_favorites_button.set_sensitive (False)
 		self.remove_from_favorites_button.set_sensitive (False)
-		if self.current_treeview_name == 'files':
-			self.refresh_button.set_sensitive (False)
-			self.page_hbox.show()
-			if self.current_movie:
-				self.page_hbox.set_sensitive (True)
-			else:
-				self.page_hbox.set_sensitive (False)
-			self.show_files_current_page()
-			self.favorites_hbox.hide()
-			self.update_progress_bar ()
-		elif self.current_treeview_name == 'movies':
+		if self.current_treeview_name == 'movies':
 			if self.movie_class_path == ():
 				self.refresh_button.set_sensitive (False)
 			elif self.movie_class_path == (CATEGORY_ID_FAVORITES,):
@@ -1649,7 +1423,6 @@ class PPStream (totem.Plugin):
 				self.page_hbox.set_sensitive (True)
 			self.page_hbox.show ()
 			self.favorites_hbox.show ()
-			self.movies_update_files_status()
 			self.update_progress_bar ()
 			if self.movie_class_path:
 				cls = self.get_movie_class(self.movie_class_path)
@@ -1670,7 +1443,6 @@ class PPStream (totem.Plugin):
 			return False
 		if self.current_treeview_name != 'categories':
 			return True
-		classes = self.ppslist.getClasses()
 		if self.fetching_classes:
 			self.progress_bar.pulse ()
 			self.progress_bar.set_text (_("Refreshing category list..."))
@@ -1702,6 +1474,20 @@ class PPStream (totem.Plugin):
 				self.progress_bar.set_text (_("Loading movie list%s...") % progressstr)
 			self.progress_bar.show ()
 			return True
+		elif self.threadpool.is_busy ():
+			remaining = 0
+			for m in cls.getMovies():
+				if not m.meta_data:
+					remaining += 1
+			num_movies = len(cls.getMovies())
+			if remaining > num_movies:
+				remaining = num_movies
+			finished = num_movies - remaining
+			progress = float(finished) / num_movies
+			self.progress_bar.set_fraction (progress)
+			progressstr = str(finished) + '/' + str(num_movies)
+			self.progress_bar.set_text (_('loading meta data(%s)...') % progressstr)
+			self.progress_bar.show ()
 		elif self.retrieveimage.is_busy ():
 			remaining = self.retrieveimage.get_remaning_tasks ()
 			num_movies = len(cls.getMovies())
@@ -1746,10 +1532,6 @@ class PPStream (totem.Plugin):
 		cls = self.get_movie_class(class_path)
 
 		movie = cls.getPage(self.movies_page)[path[0]]
-		files = movie.getFiles(self.files_page)
-		if not files:
-			return False
-
 		self.add_recent (class_path, model.get_path (iter)[0])
 		return True
 
@@ -1850,7 +1632,6 @@ class PPStream (totem.Plugin):
 			self.search.keyword = self.search_entry.get_text ()
 			self.search_terms = ''
 			self.movies_page = 0
-			self.files_page = 0
 			self.show_movies ((CATEGORY_ID_SEARCH,), True, True)
 			self.notebook.set_current_page (1)
 		else:
diff --git a/totem/plugin/ppstream.ui b/totem/plugin/ppstream.ui
index 5723c85..5fcf78e 100644
--- a/totem/plugin/ppstream.ui
+++ b/totem/plugin/ppstream.ui
@@ -168,36 +168,6 @@
             <property name="tab_fill">False</property>
           </packing>
         </child>
-        <child>
-          <object class="GtkScrolledWindow" id="pps_scrolled_window_files">
-            <property name="visible">True</property>
-            <property name="can_focus">True</property>
-            <property name="hscrollbar_policy">automatic</property>
-            <property name="vscrollbar_policy">automatic</property>
-            <child>
-              <object class="GtkTreeView" id="pps_treeview_files">
-                <property name="visible">True</property>
-                <property name="can_focus">True</property>
-                <property name="model">pps_liststore_categories</property>
-                <property name="headers_visible">False</property>
-                <property name="rules_hint">True</property>
-              </object>
-            </child>
-          </object>
-          <packing>
-            <property name="position">2</property>
-          </packing>
-        </child>
-        <child type="tab">
-          <object class="GtkLabel" id="label3">
-            <property name="visible">True</property>
-            <property name="label" translatable="yes">Files</property>
-          </object>
-          <packing>
-            <property name="position">2</property>
-            <property name="tab_fill">False</property>
-          </packing>
-        </child>
       </object>
       <packing>
         <property name="position">1</property>
diff --git a/totem/plugin/threadpool.py b/totem/plugin/threadpool.py
new file mode 100644
index 0000000..31ac724
--- /dev/null
+++ b/totem/plugin/threadpool.py
@@ -0,0 +1,240 @@
+## {{{ http://code.activestate.com/recipes/203871/ (r3)
+import threading
+from time import sleep
+
+# Ensure booleans exist (not needed for Python 2.2.1 or higher)
+try:
+    True
+except NameError:
+    False = 0
+    True = not False
+
+class Task:
+    def __init__(self, task, taskCallback, *args, **kwargs):
+        self.task = task
+        self.taskCallback = taskCallback
+        self.args = args
+        self.kwargs = kwargs
+
+class ThreadPool:
+
+    """Flexible thread pool class.  Creates a pool of threads, then
+    accepts tasks that will be dispatched to the next available
+    thread."""
+
+    def __init__(self, numThreads):
+
+        """Initialize the thread pool with numThreads workers."""
+
+        self.__threads = []
+        self.__resizeLock = threading.Condition(threading.Lock())
+        self.__taskLock = threading.Condition(threading.Lock())
+        self.__tasks = []
+        self.__isJoining = False
+        self.setThreadCount(numThreads)
+
+    def setThreadCount(self, newNumThreads):
+
+        """ External method to set the current pool size.  Acquires
+        the resizing lock, then calls the internal version to do real
+        work."""
+
+        # Can't change the thread count if we're shutting down the pool!
+        if self.__isJoining:
+            return False
+
+        self.__resizeLock.acquire()
+        try:
+            self.__setThreadCountNolock(newNumThreads)
+        finally:
+            self.__resizeLock.release()
+        return True
+
+    def __setThreadCountNolock(self, newNumThreads):
+
+        """Set the current pool size, spawning or terminating threads
+        if necessary.  Internal use only; assumes the resizing lock is
+        held."""
+
+        # If we need to grow the pool, do so
+        while newNumThreads > len(self.__threads):
+            newThread = ThreadPoolThread(self)
+            self.__threads.append(newThread)
+            newThread.start()
+        # If we need to shrink the pool, do so
+        while newNumThreads < len(self.__threads):
+            self.__threads[0].goAway()
+            del self.__threads[0]
+
+    def getThreadCount(self):
+
+        """Return the number of threads in the pool."""
+
+        self.__resizeLock.acquire()
+        try:
+            return len(self.__threads)
+        finally:
+            self.__resizeLock.release()
+
+    def queueTask(self, task, taskCallback, *args, **kwargs):
+
+        """Insert a task into the queue.  task must be callable;
+        args and taskCallback can be None."""
+
+        if self.__isJoining == True:
+            return False
+        if not callable(task):
+            return False
+
+        self.__taskLock.acquire()
+        try:
+            self.__tasks.append(Task(task, taskCallback, *args, **kwargs))
+            return True
+        finally:
+            self.__taskLock.release()
+
+    def getNextTask(self):
+
+        """ Retrieve the next task from the task queue.  For use
+        only by ThreadPoolThread objects contained in the pool."""
+
+        self.__taskLock.acquire()
+        try:
+            if self.__tasks == []:
+                return None
+            else:
+                return self.__tasks.pop(0)
+        finally:
+            self.__taskLock.release()
+
+    def joinAll(self, waitForTasks = True, waitForThreads = True):
+        """ Clear the task queue and terminate all pooled threads,
+        optionally allowing the tasks and threads to finish."""
+
+        # Mark the pool as joining to prevent any more task queueing
+        self.__isJoining = True
+
+        # Wait for tasks to finish
+        if waitForTasks:
+            while self.__tasks != []:
+                sleep(0.1)
+
+        # Tell all the threads to quit
+        self.__resizeLock.acquire()
+        try:
+            # Wait until all threads have exited
+            if waitForThreads:
+                for t in self.__threads:
+                    t.goAway()
+                for t in self.__threads:
+                    t.join()
+                    # print t,"joined"
+                    del t
+            self.__setThreadCountNolock(0)
+            self.__isJoining = True
+
+            # Reset the pool for potential reuse
+            self.__isJoining = False
+        finally:
+            self.__resizeLock.release()
+
+    def add_task(self, task, *args, **kwargs):
+        self.queueTask(task, None, *args, **kwargs)
+
+    def wait_completion(self):
+        self.joinAll()
+
+    def get_num_tasks(self):
+
+        """ Retrieve the number of tasks queued in the thread pool."""
+
+        self.__taskLock.acquire()
+        num = len(self.__tasks)
+        self.__taskLock.release()
+        return num
+
+    def is_busy(self):
+        return self.get_num_tasks() != 0
+
+class ThreadPoolThread(threading.Thread):
+
+    """ Pooled thread class. """
+
+    threadSleepTime = 0.1
+
+    def __init__(self, pool):
+
+        """ Initialize the thread and remember the pool. """
+
+        threading.Thread.__init__(self)
+        self.__pool = pool
+        self.__isDying = False
+
+    def run(self):
+
+        """ Until told to quit, retrieve the next task and execute
+        it, calling the callback if any.  """
+
+        while self.__isDying == False:
+            task = self.__pool.getNextTask()
+            # If there's nothing to do, just sleep a bit
+            if task is None:
+                sleep(ThreadPoolThread.threadSleepTime)
+            elif task.taskCallback is None:
+                task.task(*task.args, **task.kwargs)
+            else:
+                task.taskCallback(task.task(*task.args, **task.kwargs))
+
+    def goAway(self):
+
+        """ Exit the run loop next time through."""
+
+        self.__isDying = True
+
+# Usage example
+if __name__ == "__main__":
+
+    from random import randrange
+
+    # Sample task 1: given a start and end value, shuffle integers,
+    # then sort them
+
+    def sortTask(data):
+        print "SortTask starting for ", data
+        numbers = range(data[0], data[1])
+        for a in numbers:
+            rnd = randrange(0, len(numbers) - 1)
+            a, numbers[rnd] = numbers[rnd], a
+        print "SortTask sorting for ", data
+        numbers.sort()
+        print "SortTask done for ", data
+        return "Sorter ", data
+
+    # Sample task 2: just sleep for a number of seconds.
+
+    def waitTask(data):
+        print "WaitTask starting for ", data
+        print "WaitTask sleeping for %d seconds" % data
+        sleep(data)
+        return "Waiter", data
+
+    # Both tasks use the same callback
+
+    def taskCallback(data):
+        print "Callback called for", data
+
+    # Create a pool with three worker threads
+
+    pool = ThreadPool(3)
+
+    # Insert tasks into the queue and let them run
+    pool.queueTask(sortTask, taskCallback, (1000, 100000))
+    pool.queueTask(waitTask, taskCallback, 5)
+    pool.queueTask(sortTask, taskCallback, (200, 200000))
+    pool.queueTask(waitTask, taskCallback, 2)
+    pool.queueTask(sortTask, taskCallback, (3, 30000))
+    pool.queueTask(waitTask, taskCallback, 7)
+
+    # When all tasks are finished, allow the threads to terminate
+    pool.joinAll()
+## end of http://code.activestate.com/recipes/203871/ }}}