Fixed duplicates/US only issue. Also added latest albums to index page

data/css/style.css

@@ -105,6 +105,9 @@ a:active {/*colour in NN4.xx is red*/
   	color: #5E2612; 
   	text-decoration: underline; 
 	}
+a.gray {
+	color: #CFCFCF
+	}
 a.external {
 	color: blue;
 	font-size:12px;
@@ -112,6 +115,9 @@ a.external {
 a.blue {
 	color: blue;
 	}
+a.green {
+	color: #629632;
+	}
 a.externalred {
 	color: red;
 	font-size:12px;

helpers.py

@@ -0,0 +1,11 @@
+def multikeysort(items, columns):
+    from operator import itemgetter
+    comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns]  
+    def comparer(left, right):
+        for fn, mult in comparers:
+            result = cmp(fn(left), fn(right))
+            if result:
+                return mult * result
+        else:
+            return 0
+    return sorted(items, cmp=comparer)

itunesimport.py

@@ -3,6 +3,7 @@ from configobj import ConfigObj
 import musicbrainz2.webservice as ws
 import musicbrainz2.model as m
 import musicbrainz2.utils as u
+from mb import getReleaseGroup
 import string
 import time
 import os
@@ -69,7 +70,6 @@ def itunesImport(pathtoxml):
 def importartist(artistlist):
 	for name in artistlist:
 		logger.log(u"Querying MusicBrainz for: "+name)
-		time.sleep(1)
 		artistResults = ws.Query().getArtists(ws.ArtistFilter(string.replace(name, '&', '%38'), limit=1))		
 		for result in artistResults:
 			if result.artist.name == 'Various Artists':
@@ -78,7 +78,7 @@ def importartist(artistlist):
 				logger.log(u"Found best match: "+result.artist.name+". Gathering album information...")
 				time.sleep(1)
 				artistid = u.extractUuid(result.artist.id)
-				inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), ratings=False, releaseGroups=False)
+				inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), releaseGroups=True)
 				artist = ws.Query().getArtistById(artistid, inc)
 				conn=sqlite3.connect(database)
 				c=conn.cursor()
@@ -88,31 +88,31 @@ def importartist(artistlist):
 					logger.log(result.artist.name + u" is already in the database, skipping")
 				else:
 					c.execute('INSERT INTO artists VALUES( ?, ?, ?, CURRENT_DATE, ?)', (artistid, artist.name, artist.sortName, 'Active'))
-					for release in artist.getReleases():
+					for rg in artist.getReleaseGroups():
-						time.sleep(1)
+						rgid = u.extractUuid(rg.id)
-						releaseid = u.extractUuid(release.id)
+						
+						releaseid = getReleaseGroup(rgid)
+						
 						inc = ws.ReleaseIncludes(artist=True, releaseEvents= True, tracks= True, releaseGroup=True)
 						results = ws.Query().getReleaseById(releaseid, inc)
+
+						logger.log(u"Now adding album: " + results.title+ " to the database")
+						c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
+						conn.commit()
+						c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
+								
+						latestrelease = c.fetchall()
 						
-						for event in results.releaseEvents:
+						if latestrelease[0][0] > latestrelease[0][1]:
+							logger.log(results.title + u" is an upcoming album. Setting its status to 'Wanted'...")
+							c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
+						else:
+							pass
 							
-							if event.country == 'US':
+						for track in results.tracks:
-								
+							c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
-								c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
+						time.sleep(1)
-								conn.commit()
+				time.sleep(1)
-								c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
+
-								
+				conn.commit()		
-								latestrelease = c.fetchall()
-								
-								if latestrelease[0][0] > latestrelease[0][1]:
-									c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
-								else:
-									pass
-								for track in results.tracks:
-									c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
-								conn.commit()
-	
-							else:
-								logger.log(results.title + u" is not a US release. Skipping for now")
-			
 				c.close()

mb.py

@@ -0,0 +1,84 @@
+import time
+
+import musicbrainz2.webservice as ws
+import musicbrainz2.model as m
+import musicbrainz2.utils as u
+
+from musicbrainz2.webservice import WebServiceError
+
+from helpers import multikeysort
+
+q = ws.Query()
+
+
+def findArtist(name, limit=1):
+
+	artistlist = []
+	
+	artistResults = q.getArtists(ws.ArtistFilter(name=name, limit=limit))
+	
+	for result in artistResults:
+	
+		artistid = u.extractUuid(result.artist.id)
+		artistlist.append([result.artist.name, artistid])
+		
+	return artistlist
+
+def getArtist(artistid):
+
+
+	rglist = []
+
+	#Get all official release groups
+	inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), ratings=False, releaseGroups=True)
+	artist = q.getArtistById(artistid, inc)
+	
+	for rg in artist.getReleaseGroups():
+		
+		rgid = u.extractUuid(rg.id)
+		rglist.append([rg.title, rgid])
+	
+	return rglist
+	
+def getReleaseGroup(rgid):
+
+	releaselist = []
+	
+	inc = ws.ReleaseGroupIncludes(releases=True)
+	releaseGroup = q.getReleaseGroupById(rgid, inc)
+	
+	# I think for now we have to make separate queries for each release, in order
+	# to get more detailed release info (ASIN, track count, etc.)
+	for release in releaseGroup.releases:
+			
+		releaseid = u.extractUuid(release.id)
+		inc = ws.ReleaseIncludes(tracks=True)
+		
+		releaseResult = q.getReleaseById(releaseid, inc)
+		
+		release_dict = {
+			'asin':			bool(releaseResult.asin),
+			'tracks':		len(releaseResult.getTracks()),
+			'releaseid':	u.extractUuid(releaseResult.id)
+		}
+		
+		releaselist.append(release_dict)
+		time.sleep(1)
+
+	a = multikeysort(releaselist, ['-asin', '-tracks'])
+
+	releaseid = a[0]['releaseid']
+	
+	return releaseid
+	
+def getRelease(releaseid):
+	"""
+	Given a release id, gather all the info and return it as a list
+	"""
+	inc = ws.ReleaseIncludes(artist=True, tracks=True, releaseGroup=True)
+	release = q.getReleaseById(releaseid, inc)
+	
+	releasedetail = []
+	
+	releasedetail.append(release.id)
+

updater.py

@@ -2,6 +2,7 @@ from webServer import database
 import musicbrainz2.webservice as ws
 import musicbrainz2.model as m
 import musicbrainz2.utils as u
+from mb import getReleaseGroup
 import sqlite3
 import time
 
@@ -26,53 +27,47 @@ def dbUpdate():
 		c.execute('SELECT AlbumID from albums WHERE ArtistID="%s"' % artistid)
 		albumlist = c.fetchall()
 		
-		inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), ratings=False, releaseGroups=False)
+		inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), releaseGroups=True)
 		artist = ws.Query().getArtistById(artistid, inc)
 		
-		for release in artist.getReleases():
+		for rg in artist.getReleaseGroups():
 			
-			releaseid = u.extractUuid(release.id)
+			rgid = u.extractUuid(rg.id)
+			releaseid = getReleaseGroup(rgid)
 			inc = ws.ReleaseIncludes(artist=True, releaseEvents= True, tracks= True, releaseGroup=True)
 			results = ws.Query().getReleaseById(releaseid, inc)
-			time.sleep(2)
 			
-			for event in results.releaseEvents:
+			if any(releaseid in x for x in albumlist):
-				
-				if event.country == 'US':
 					
-					if any(releaseid in x for x in albumlist):
+				logger.log(results.title + " already exists in the database. Updating ASIN, Release Date, Tracks")
-					
-						logger.log(results.title + " already exists in the database. Updating ASIN, Release Date, Tracks")
 						
-						c.execute('UPDATE albums SET AlbumASIN="%s", ReleaseDate="%s" WHERE AlbumID="%s"' % (results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id)))
+				c.execute('UPDATE albums SET AlbumASIN="%s", ReleaseDate="%s" WHERE AlbumID="%s"' % (results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id)))
 		
-						for track in results.tracks:
+				for track in results.tracks:
-							c.execute('UPDATE tracks SET TrackDuration="%s" WHERE AlbumID="%s" AND TrackID="%s"' % (track.duration, u.extractUuid(results.id), u.extractUuid(track.id)))
+					c.execute('UPDATE tracks SET TrackDuration="%s" WHERE AlbumID="%s" AND TrackID="%s"' % (track.duration, u.extractUuid(results.id), u.extractUuid(track.id)))
-							conn.commit()
+					conn.commit()
 						
-					else:
+			else:
+				
+				logger.log(u"New album found! Adding "+results.title+"to the database...")
+				c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
+				conn.commit()
+				c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
 						
-						logger.log(u"New album found! Adding "+results.title+"to the database...")
+				latestrelease = c.fetchall()
-						c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
-						conn.commit()
-						c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
 						
-						latestrelease = c.fetchall()
+				if latestrelease[0][0] > latestrelease[0][1]:
-						
-						if latestrelease[0][0] > latestrelease[0][1]:
 							
-							c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
+					c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
-						
-						else:
-							pass
-						
-						for track in results.tracks:
-							
-							c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
-							conn.commit()
 						
 				else:
-					logger.log(results.title + " is not a US release. Skipping it for now")
+					pass
+						
+				for track in results.tracks:
+							
+					c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
+					conn.commit()
+			time.sleep(1)
 		i += 1
 	
 	conn.commit()

webServer.py

@@ -12,6 +12,7 @@ import sqlite3
 import sys
 import configobj
 from headphones import FULL_PATH, config_file
+from mb import getReleaseGroup
 import logger
 
 database = os.path.join(FULL_PATH, 'headphones.db')
@@ -46,10 +47,10 @@ class Headphones:
 				today = datetime.date.today()
 				if len(latestalbum) > 0:
 					if latestalbum[0][1] > datetime.date.isoformat(today):
-						newalbumName = '<font color="#5DFC0A" size="3px"><a href="albumPage?AlbumID=%s"><i>%s</i>' % (latestalbum[0][3], latestalbum[0][0])
+						newalbumName = '<a class="green" href="albumPage?AlbumID=%s"><i><b>%s</b></i>' % (latestalbum[0][3], latestalbum[0][0])
-						releaseDate = '(%s)</a></font>' % latestalbum[0][1]
+						releaseDate = '(%s)</a>' % latestalbum[0][1]
 					else:
-						newalbumName = '<font color="#CFCFCF">None</font>'
+						newalbumName = '<a class="gray" href="albumPage?AlbumID=%s"><i>%s</i>' % (latestalbum[0][3], latestalbum[0][0])
 						releaseDate = ""
 				if len(latestalbum) == 0:
 						newalbumName = '<font color="#CFCFCF">None</font>'
@@ -201,7 +202,7 @@ class Headphones:
 	artistInfo.exposed = True
 
 	def addArtist(self, artistid):
-		inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), ratings=False, releaseGroups=False)
+		inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, m.Release.TYPE_ALBUM), releaseGroups=True)
 		artist = ws.Query().getArtistById(artistid, inc)
 		conn=sqlite3.connect(database)
 		c=conn.cursor()
@@ -217,30 +218,28 @@ class Headphones:
 		else:
 			logger.log(u"Adding " + artist.name + " to the database.")
 			c.execute('INSERT INTO artists VALUES( ?, ?, ?, CURRENT_DATE, ?)', (artistid, artist.name, artist.sortName, 'Active'))
-			
+			for rg in artist.getReleaseGroups():
-			for release in artist.getReleases():
+				rgid = u.extractUuid(rg.id)
-				releaseid = u.extractUuid(release.id)
+				
+				releaseid = getReleaseGroup(rgid)
+				
 				inc = ws.ReleaseIncludes(artist=True, releaseEvents= True, tracks= True, releaseGroup=True)
 				results = ws.Query().getReleaseById(releaseid, inc)
-				time.sleep(1)
 				
-				for event in results.releaseEvents:
+				logger.log(u"Now adding album: " + results.title+ " to the database")
-					if event.country == 'US':
+				c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
-						logger.log(u"Now adding album: " + results.title+ " to the database")
+				c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
-						c.execute('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', (artistid, results.artist.name, results.title, results.asin, results.getEarliestReleaseDate(), u.extractUuid(results.id), 'Skipped'))
+				latestrelease = c.fetchall()
-						c.execute('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID="%s"' % u.extractUuid(results.id))
-						latestrelease = c.fetchall()
 						
-						if latestrelease[0][0] > latestrelease[0][1]:
+				if latestrelease[0][0] > latestrelease[0][1]:
-							logger.log(results.title + u" is an upcoming album. Setting its status to 'Wanted'...")
+					logger.log(results.title + u" is an upcoming album. Setting its status to 'Wanted'...")
-							c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
+					c.execute('UPDATE albums SET Status = "Wanted" WHERE AlbumID="%s"' % u.extractUuid(results.id))
-						else:
+				else:
-							pass
+					pass
-						
+					
-						for track in results.tracks:
+				for track in results.tracks:
-							c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
+					c.execute('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', (artistid, results.artist.name, results.title, results.asin, u.extractUuid(results.id), track.title, track.duration, u.extractUuid(track.id)))
-					else:
+				time.sleep(1)
-						logger.log(results.title + " is not a US release. Skipping it for now", logger.DEBUG)
 			
 			conn.commit()
 			c.close()