summaryrefslogtreecommitdiff
path: root/invest-applet/invest/quotes.py
blob: 5331e0078ab1c13f40a2024dc81819bea3ba72bf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
from os.path import join, getmtime

import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
from gi.repository import Gdk
from gi.repository import GdkPixbuf
from gi.repository import GObject
from gettext import gettext as _
import csv
import locale
from urllib import urlopen
import datetime
from threading import Thread

import mate_invest, mate_invest.about, mate_invest.chart
import currencies

CHUNK_SIZE = 512*1024 # 512 kB
AUTOREFRESH_TIMEOUT = 15*60*1000 # 15 minutes

QUOTES_URL="http://finance.yahoo.com/d/quotes.csv?s=%(s)s&f=snc4l1d1t1c1ohgv&e=.csv"

# Sample (09/2/2010): "UCG.MI","UNICREDIT","EUR","UCG.MI",1.9410,"2/9/2010","6:10am",+0.0210,1.9080,1.9810,1.8920,166691232
QUOTES_CSV_FIELDS=["ticker", "label", "currency", ("trade", float), "date", "time", ("variation", float), ("open", float), ("high", float), ("low", float), ("volume", int)]

# based on http://www.johnstowers.co.nz/blog/index.php/2007/03/12/threading-and-pygtk/
class _IdleObject(GObject.GObject):
	"""
	Override GObject.GObject to always emit signals in the main thread
	by emmitting on an idle handler
	"""
	def __init__(self):
		GObject.GObject.__init__(self)

	def emit(self, *args):
		GObject.idle_add(GObject.GObject.emit,self,*args)

class QuotesRetriever(Thread, _IdleObject):
	"""
	Thread which uses gobject signals to return information
	to the GUI.
	"""
	__gsignals__ =  {
			"completed": (
				GObject.SignalFlags.RUN_LAST, None, []),
			# FIXME: We don't monitor progress, yet ...
			#"progress": (
			#	GObject.SignalFlags.RUN_LAST, None, [
			#	GObject.TYPE_FLOAT])        #percent complete
			}

	def __init__(self, tickers):
		Thread.__init__(self)
		_IdleObject.__init__(self)
		self.tickers = tickers
		self.retrieved = False
		self.data = []
		self.currencies = []
                mate_invest.debug("QuotesRetriever created");

	def run(self):
                mate_invest.debug("QuotesRetriever started");
		quotes_url = QUOTES_URL % {"s": self.tickers}
		try:
			quotes_file = urlopen(quotes_url, proxies = mate_invest.PROXY)
			self.data = quotes_file.read ()
			quotes_file.close ()
		except Exception, msg:
			mate_invest.debug("Error while retrieving quotes data (url = %s): %s" % (quotes_url, msg))
		else:
			self.retrieved = True
		self.emit("completed")


class QuoteUpdater(Gtk.ListStore):
	updated = False
	last_updated = None
	quotes_valid = False
	timeout_id = None
	SYMBOL, LABEL, CURRENCY, TICKER_ONLY, BALANCE, BALANCE_PCT, VALUE, VARIATION_PCT, PB = range(9)
	def __init__ (self, change_icon_callback, set_tooltip_callback):
		Gtk.ListStore.__init__ (self, GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING, bool, float, float, float, float, GdkPixbuf.Pixbuf)
		self.set_update_interval(AUTOREFRESH_TIMEOUT)
		self.change_icon_callback = change_icon_callback
		self.set_tooltip_callback = set_tooltip_callback
		self.set_sort_column_id(1, Gtk.SortType.ASCENDING)
		self.load_quotes()	# read the last cached quotes file
		self.refresh()	# download a new quotes file, this may fail if disconnected

		# tell the network manager to notify me when network status changes
		mate_invest.nm.set_statechange_callback(self.nm_state_changed)

	# loads the cached csv file and its last-modification-time as self.last_updated
	def load_quotes(self):
		mate_invest.debug("Loading quotes");
		try:
			f = open(mate_invest.QUOTES_FILE, 'r')
			data = f.readlines()
			f.close()

			self.populate(self.parse_yahoo_csv(csv.reader(data)))
			self.updated = True
			self.last_updated = datetime.datetime.fromtimestamp(getmtime(mate_invest.QUOTES_FILE))
			self.update_tooltip()
		except Exception, msg:
			mate_invest.error("Could not load the cached quotes file %s: %s" % (mate_invest.QUOTES_FILE, msg) )

	# stores the csv content on disk so it can be used on next start up
	def save_quotes(self, data):
		mate_invest.debug("Storing quotes")
		try:
			f = open(mate_invest.QUOTES_FILE, 'w')
			f.write(data)
			f.close()
		except Exception, msg:
			mate_invest.error("Could not save the retrieved quotes file to %s: %s" % (mate_invest.QUOTES_FILE, msg) )

	def set_update_interval(self, interval):
		if self.timeout_id != None:
			mate_invest.debug("Canceling refresh timer")
			GObject.source_remove(self.timeout_id)
			self.timeout_id = None
		if interval > 0:
			mate_invest.debug("Setting refresh timer to %s:%02d.%03d" % ( interval / 60000, interval % 60000 / 1000, interval % 1000) )
			self.timeout_id = GObject.timeout_add(interval, self.refresh)

	def nm_state_changed(self):
		# when nm is online but we do not have an update timer, create it and refresh
		if mate_invest.nm.online():
			if self.timeout_id == None:
				self.set_update_interval(AUTOREFRESH_TIMEOUT)
				self.refresh()

	def refresh(self):
		mate_invest.debug("Refreshing")

		# when nm tells me I am offline, stop the update interval
		if mate_invest.nm.offline():
			mate_invest.debug("We are offline, stopping update timer")
			self.set_update_interval(0)
			return False

		if len(mate_invest.STOCKS) == 0:
                        mate_invest.debug("No stocks configured")
			return True

		tickers = '+'.join(mate_invest.STOCKS.keys())
                mate_invest.debug("creating QuotesRetriever")
		quotes_retriever = QuotesRetriever(tickers)
		quotes_retriever.connect("completed", self.on_retriever_completed)
                mate_invest.debug("starting QuotesRetriever")
		quotes_retriever.start()
                mate_invest.debug("started QuotesRetriever")

		return True


	# locale-aware formatting of the percent float (decimal point, thousand grouping point) with 2 decimal digits
	def format_percent(self, value):
		return locale.format("%+.2f", value, True) + "%"

	# locale-aware formatting of the float value (decimal point, thousand grouping point) with sign and 2 decimal digits
	def format_difference(self, value):
		return locale.format("%+.2f", value, True, True)

	def on_retriever_completed(self, retriever):
		if retriever.retrieved == False:
                        mate_invest.debug("QuotesRetriever failed");
                        self.update_tooltip(_('Invest could not connect to Yahoo! Finance'))

		else:
                        mate_invest.debug("QuotesRetriever completed");
                        # cache the retrieved csv file
                        self.save_quotes(retriever.data)
                        # load the cache and parse it
                        self.load_quotes()

	def on_currency_retriever_completed(self, retriever):
		if retriever.retrieved == False:
			mate_invest.error("Failed to retrieve currency rates!")
		else:
			mate_invest.debug("CurrencyRetriever completed")
			self.save_currencies(retriever.data)
			self.load_currencies()
		self.update_tooltip()

	def save_currencies(self, data):
		mate_invest.debug("Storing currencies to %s" % mate_invest.CURRENCIES_FILE)
		try:
			f = open(mate_invest.CURRENCIES_FILE, 'w')
			f.write(data)
			f.close()
		except Exception as msg:
			mate_invest.error("Could not save the retrieved currencies to %s: %s" % (mate_invest.CURRENCIES_FILE, msg) )

	def load_currencies(self):
		mate_invest.debug("Loading currencies from %s" % mate_invest.CURRENCIES_FILE)
		try:
			f = open(mate_invest.CURRENCIES_FILE, 'r')
			data = f.readlines()
			f.close()

			self.convert_currencies(self.parse_yahoo_csv(csv.reader(data)))
		except Exception as msg:
			mate_invest.error("Could not load the currencies from %s: %s" % (mate_invest.CURRENCIES_FILE, msg) )

	def update_tooltip(self, msg = None):
		tooltip = []
		if self.quotes_count > 0:
			# Translators: This is share-market jargon. It is the average percentage change of all stock prices. The %s gets replaced with the string value of the change (localized), including the percent sign.
			tooltip.append(_('Average change: %s') % self.format_percent(self.avg_quotes_change))
		for currency, stats in self.statistics.items():
			# get the statsitics
			balance = stats["balance"]
			paid = stats["paid"]
			change = self.format_percent(balance / paid * 100)
			balance = self.format_difference(balance)

			# Translators: This is share-market jargon. It refers to the total difference between the current price and purchase price for all the shares put together for a particular currency. i.e. How much money would be earned if they were sold right now. The first string is the change value, the second the currency, and the third value is the percentage of the change, formatted using user's locale.
			tooltip.append(_('Positions balance: %s %s (%s)') % (balance, currency, change))
		if self.last_updated != None:
			tooltip.append(_('Updated at %s') % self.last_updated.strftime("%H:%M"))
		if msg != None:
			tooltip.append(msg)
		self.set_tooltip_callback('\n'.join(tooltip))


	def parse_yahoo_csv(self, csvreader):
		result = {}
		for fields in csvreader:
			if len(fields) == 0:
				continue

			result[fields[0]] = {}
			for i, field in enumerate(QUOTES_CSV_FIELDS):
				if type(field) == tuple:
					try:
						result[fields[0]][field[0]] = field[1](fields[i])
					except:
						result[fields[0]][field[0]] = 0
				else:
					result[fields[0]][field] = fields[i]
			# calculated fields
			try:
				result[fields[0]]['variation_pct'] = result[fields[0]]['variation'] / float(result[fields[0]]['trade'] - result[fields[0]]['variation']) * 100
			except ZeroDivisionError:
				result[fields[0]]['variation_pct'] = 0
		return result

	# Computes the balance of the given purchases using a certain current value
	# and optionally a current exchange rate.
	def balance(self, purchases, value, currentrate=None):
		current = 0
		paid = 0

		for purchase in purchases:
			if purchase["amount"] != 0:
				buyrate = purchase["exchange"]
				# if the buy rate is invalid, use 1.0
				if buyrate <= 0:
					buyrate = 1.0

				# if no current rate is given, ignore buy rate
				if currentrate == None:
					buyrate = 1.0
					rate = 1.0
				else:
					# otherwise, take use buy rate and current rate to compute the balance
					rate = currentrate

				# current value is the current rate * amount * value
				current += rate * purchase["amount"] * value
				# paid is buy rate * ( amount * price + commission )
				paid += buyrate * (purchase["amount"] * purchase["bought"] + purchase["comission"])

		balance = current - paid
		if paid != 0:
			change = 100*balance/paid
		else:
			change = 100 # Not technically correct, but it will look more intuitive than the real result of infinity.

		return (balance, change)

	def populate(self, quotes):
		if (len(quotes) == 0):
			return

		self.clear()
		self.currencies = []

		try:
			quote_items = quotes.items ()
			quote_items.sort ()

			quotes_change = 0
			self.quotes_count = 0
			self.statistics = {}

			for ticker, val in quote_items:
				pb = None

                                # ignore unknown stocks
                                if ticker not in mate_invest.STOCKS.keys():
                                    mate_invest.debug("Observed unknown stock: %s" % ticker)
                                    continue

				# get the label of this stock for later reuse
				label = mate_invest.STOCKS[ticker]["label"]
				if len(label) == 0:
					if len(val["label"]) != 0:
						label = val["label"]
					else:
						label = ticker

				# make sure the currency field is upper case
				val["currency"] = val["currency"].upper();

				# the currency of currency conversion rates like EURUSD=X is wrong in csv
				# this can be fixed easily by reusing the latter currency in the symbol
				if len(ticker) == 8 and ticker.endswith("=X"):
					val["currency"] = ticker[3:6]

				# indices should not have a currency, though yahoo says so
				if ticker.startswith("^"):
					val["currency"] = ""

				# sometimes, funny currencies are returned (special characters), only consider known currencies
				if len(val["currency"]) > 0 and val["currency"] not in currencies.Currencies.currencies:
					mate_invest.debug("Currency '%s' is not known, dropping" % val["currency"])
					val["currency"] = ""

				# if this is a currency not yet seen and different from the target currency, memorize it
				if val["currency"] not in self.currencies and len(val["currency"]) > 0:
					self.currencies.append(val["currency"])

				# Check whether the symbol is a simple quote, or a portfolio value
				is_simple_quote = True
				for purchase in mate_invest.STOCKS[ticker]["purchases"]:
					if purchase["amount"] != 0:
						is_simple_quote = False
						break

				if is_simple_quote:
					row = self.insert(0, [ticker, label, val["currency"], True, 0.0, 0.0, val["trade"], val["variation_pct"], pb])
				else:
					(balance, change) = self.balance(mate_invest.STOCKS[ticker]["purchases"], val["trade"])
					row = self.insert(0, [ticker, label, val["currency"], False, float(balance), float(change), val["trade"], val["variation_pct"], pb])
					self.add_balance_change(balance, change, val["currency"])

				if len(ticker.split('.')) == 2:
					url = 'http://chart.finance.yahoo.com/h?s=%s' % ticker
				else:
					url = 'http://ichart.yahoo.com/h?s=%s' % ticker

				image_retriever = mate_invest.chart.ImageRetriever(url)
				image_retriever.connect("completed", self.set_pb_callback, row)
				image_retriever.start()

			        quotes_change += val['variation_pct']
			        self.quotes_count += 1

			# we can only compute an avg quote if there are quotes
			if self.quotes_count > 0:
			        self.avg_quotes_change = quotes_change / float(self.quotes_count)

			        # change icon
			        quotes_change_sign = 0
			        if self.avg_quotes_change != 0:
			                quotes_change_sign = self.avg_quotes_change / abs(self.avg_quotes_change)
			        self.change_icon_callback(quotes_change_sign)
			else:
				self.avg_quotes_change = 0
				
			# mark quotes to finally be valid
			self.quotes_valid = True

		except Exception, msg:
			mate_invest.debug("Failed to populate quotes: %s" % msg)
			mate_invest.debug(quotes)
			self.quotes_valid = False

		# start retrieving currency conversion rates
		if mate_invest.CONFIG.has_key("currency"):
			target_currency = mate_invest.CONFIG["currency"]
			symbols = []

			mate_invest.debug("These currencies occur: %s" % self.currencies)
			for currency in self.currencies:
				if currency == target_currency:
					continue

				mate_invest.debug("%s will be converted to %s" % ( currency, target_currency ))
				symbol = currency + target_currency + "=X"
				symbols.append(symbol)

			if len(symbols) > 0:
				tickers = '+'.join(symbols)
				quotes_retriever = QuotesRetriever(tickers)
				quotes_retriever.connect("completed", self.on_currency_retriever_completed)
				quotes_retriever.start()

	def convert_currencies(self, quotes):
		# if there is no target currency, this method should never have been called
		if not mate_invest.CONFIG.has_key("currency"):
			return

		# reset the overall balance
		self.statistics = {}

		# collect the rates for the currencies
		rates = {}
		for symbol, data in quotes.items():
			currency = symbol[0:3]
			rate = data["trade"]
			rates[currency] = rate

		# convert all non target currencies
		target_currency = mate_invest.CONFIG["currency"]
		iter = self.get_iter_first()
		while iter != None:
			currency = self.get_value(iter, self.CURRENCY)
			symbol = self.get_value(iter, self.SYMBOL)
			# ignore stocks that are currency conversions
			# and only convert stocks that are not in the target currency
			# and if we have a conversion rate
			if not ( len(symbol) == 8 and symbol[6:8] == "=X" ) and \
			   currency != target_currency and rates.has_key(currency):
				# first convert the balance, it needs the original value
				if not self.get_value(iter, self.TICKER_ONLY):
					ticker = self.get_value(iter, self.SYMBOL)
					value = self.get_value(iter, self.VALUE)
					(balance, change) = self.balance(mate_invest.STOCKS[ticker]["purchases"], value, rates[currency])
					self.set_value(iter, self.BALANCE, balance)
					self.set_value(iter, self.BALANCE_PCT, change)
					self.add_balance_change(balance, change, target_currency)

				# now, convert the value
				value = self.get_value(iter, self.VALUE)
				value *= rates[currency]
				self.set_value(iter, self.VALUE, value)
				self.set_value(iter, self.CURRENCY, target_currency)

			else:
				# consider non-converted stocks here
				balance = self.get_value(iter, self.BALANCE)
				change  = self.get_value(iter, self.BALANCE_PCT)
				self.add_balance_change(balance, change, currency)

			iter = self.iter_next(iter)

	def add_balance_change(self, balance, change, currency):
		if balance == 0 and change == 0:
			return

		if self.statistics.has_key(currency):
			self.statistics[currency]["balance"] += balance
			self.statistics[currency]["paid"] += balance/change*100
		else:
			self.statistics[currency] = { "balance" : balance, "paid" : balance/change*100 }

	def set_pb_callback(self, retriever, row):
		self.set_value(row, self.PB, retriever.image.get_pixbuf())

	# check if we have only simple quotes
	def simple_quotes_only(self):
		res = True
		for entry, data in mate_invest.STOCKS.iteritems():
			purchases = data["purchases"]
			for purchase in purchases:
				if purchase["amount"] != 0:
					res = False
					break
		return res