improved image cache, new example

addons/gift/gift_node.gd

@@ -28,10 +28,10 @@ signal emote_downloaded(emote_id)
 # Badge has been downloaded
 signal badge_downloaded(badge_name)
 
-# Messages starting with one of these symbols are handled. '/' will be ignored, reserved by Twitch.
+# Messages starting with one of these symbols are handled as commands. '/' will be ignored, reserved by Twitch.
 export(Array, String) var command_prefixes : Array = ["!"]
-# Time to wait after each sent chat message. Values below ~0.31 might lead to a disconnect after 100 messages.
-export(float) var chat_timeout = 0.32
+# Time to wait in msec after each sent chat message. Values below ~310 might lead to a disconnect after 100 messages.
+export(int) var chat_timeout_ms = 320
 export(bool) var get_images : bool = false
 # If true, caches emotes/badges to disk, so that they don't have to be redownloaded on every restart.
 # This however means that they might not be updated if they change until you clear the cache.
@@ -39,12 +39,12 @@ export(bool) var disk_cache : bool = false
 # Disk Cache has to be enbaled for this to work
 export(String, FILE) var disk_cache_path = "user://gift/cache"
 
-var websocket : WebSocketClient = WebSocketClient.new()
-var user_regex = RegEx.new()
+var websocket := WebSocketClient.new()
+var user_regex := RegEx.new()
 var twitch_restarting
 # Twitch disconnects connected clients if too many chat messages are being sent. (At about 100 messages/30s)
 var chat_queue = []
-onready var chat_accu = chat_timeout
+var last_msg = OS.get_ticks_msec()
 # Mapping of channels to their channel info, like available badges.
 var channels : Dictionary = {}
 var commands : Dictionary = {}
@@ -81,7 +81,7 @@ func _ready() -> void:
 	websocket.connect("connection_error", self, "connection_error")
 	if(get_images):
 		image_cache = ImageCache.new(disk_cache, disk_cache_path)
-		add_child(image_cache)
+#		add_child(image_cache)
 
 func connect_to_twitch() -> void:
 	if(websocket.connect_to_url("wss://irc-ws.chat.twitch.tv:443") != OK):
@@ -91,11 +91,9 @@ func connect_to_twitch() -> void:
 func _process(delta : float) -> void:
 	if(websocket.get_connection_status() != NetworkedMultiplayerPeer.CONNECTION_DISCONNECTED):
 		websocket.poll()
-		if(!chat_queue.empty() && chat_accu >= chat_timeout):
+		if (!chat_queue.empty() && (last_msg + chat_timeout_ms) <= OS.get_ticks_msec()):
 			send(chat_queue.pop_front())
-			chat_accu = 0
-		else:
-			chat_accu += delta
+			last_msg = OS.get_ticks_msec()
 
 # Login using a oauth token.
 # You will have to either get a oauth token yourself or use
@@ -129,7 +127,7 @@ func chat(message : String, channel : String = ""):
 	else:
 		print_debug("No channel specified.")
 
-func whisper(message : String, target : String):
+func whisper(message : String, target : String) -> void:
 	chat("/w " + target + " " + message)
 
 func data_received() -> void:
@@ -194,13 +192,6 @@ func handle_message(message : String, tags : Dictionary) -> void:
 			var sender_data : SenderData = SenderData.new(user_regex.search(msg[0]).get_string(), msg[2], tags)
 			handle_command(sender_data, msg)
 			emit_signal("chat_message", sender_data, msg[3].right(1))
-			if(get_images):
-				if(!image_cache.badge_map.has(tags["room-id"])):
-					image_cache.get_badge_mappings(tags["room-id"])
-				for emote in tags["emotes"].split("/", false):
-					image_cache.get_emote(emote.split(":")[0])
-				for badge in tags["badges"].split(",", false):
-					image_cache.get_badge(badge, tags["room-id"])
 		"WHISPER":
 			var sender_data : SenderData = SenderData.new(user_regex.search(msg[0]).get_string(), msg[2], tags)
 			handle_command(sender_data, msg, true)

addons/gift/placeholder.png.import

@@ -0,0 +1,13 @@
+[remap]
+
+importer="image"
+type="Image"
+path="res://.import/placeholder.png-7d8d01cafa2d5188ea51e03e3fda7124.image"
+
+[deps]
+
+source_file="res://addons/gift/placeholder.png"
+dest_files=[ "res://.import/placeholder.png-7d8d01cafa2d5188ea51e03e3fda7124.image" ]
+
+[params]
+

addons/gift/util/cmd_data.gd

@@ -13,4 +13,4 @@ func _init(f_ref : FuncRef, perm_lvl : int, mx_args : int, mn_args : int, whr :
 	max_args = mx_args
 	min_args = mn_args
 	where = whr
-	
+	

addons/gift/util/cmd_info.gd

@@ -9,4 +9,4 @@ func _init(sndr_dt, cmd, whspr):
 	sender_data = sndr_dt
 	command = cmd
 	whisper = whspr
-	
+	

addons/gift/util/image_cache.gd

@@ -1,111 +1,197 @@
-extends Node
+extends Resource
 class_name ImageCache
 
-signal badge_mapping_available
+enum RequestType {
+	EMOTE,
+	BADGE,
+	BADGE_MAPPING
+}
 
-var cached_images : Dictionary = {"emotes": {}, "badges": {}}
-var cache_mutex = Mutex.new()
-var badge_map : Dictionary = {}
-var badge_mutex = Mutex.new()
-var dl_queue : PoolStringArray = []
-var disk_cache : bool
-var disk_cache_path : String
+var caches := {
+	RequestType.EMOTE: {},
+	RequestType.BADGE: {},
+	RequestType.BADGE_MAPPING: {}
+}
+
+var queue := []
+var thread := Thread.new()
+var mutex := Mutex.new()
+var active = true
+var http_client := HTTPClient.new()
+var host : String
 
 var file : File = File.new()
 var dir : Directory = Directory.new()
+var cache_path : String
+var disk_cache : bool
+
+const HEADERS : PoolStringArray = PoolStringArray([
+	"User-Agent: GIFT/1.0 (Godot Engine)",
+	"Accept: */*"
+])
 
 func _init(do_disk_cache : bool, cache_path : String) -> void:
-	disk_cache = do_disk_cache
-	disk_cache_path = cache_path
+	self.disk_cache = do_disk_cache
+	self.cache_path = cache_path
+	thread.start(self, "start")
 
-func _ready() -> void:
-	if(disk_cache):
-		for cache_dir in cached_images.keys():
-			cached_images[cache_dir] = {}
-			dir.make_dir_recursive(disk_cache_path + "/" + cache_dir)
-			dir.open(disk_cache_path + "/" + cache_dir)
-			dir.list_dir_begin(true)
-			var current = dir.get_next()
-			while current != "":
-				if(!dir.current_is_dir()):
-					file.open(dir.get_current_dir() + "/" + current, File.READ)
-					var img : Image = Image.new()
-					img.load_png_from_buffer(file.get_buffer(file.get_len()))
-					file.close()
-					var img_texture : ImageTexture = ImageTexture.new()
-					img_texture.create_from_image(img, 0)
-					cache_mutex.lock()
-					cached_images[cache_dir][current.get_basename()] = img_texture
-					cache_mutex.unlock()
-				current = dir.get_next()
-		dir.open(disk_cache_path)
-		dir.list_dir_begin(true)
-		var current = dir.get_next()
-		while current != "":
-			if(!dir.current_is_dir()):
-				file.open(disk_cache_path + "/" + current, File.READ)
-				badge_map[current.get_basename()] = parse_json(file.get_as_text())["badge_sets"]
+func start(params) -> void:
+	var f : File = File.new()
+	var d : Directory = Directory.new()
+	if (disk_cache):
+		for type in caches.keys():
+			var cache_dir = RequestType.keys()[type]
+			caches[cache_dir] = {}
+			var error := d.make_dir_recursive(cache_path + "/" + cache_dir)
+	while active:
+		if (!queue.empty()):
+			mutex.lock()
+			var entry : Entry = queue.pop_front()
+			mutex.unlock()
+			var buffer : PoolByteArray = http_request(entry.path, entry.type)
+			if (disk_cache):
+				if !d.dir_exists(entry.filename.get_base_dir()):
+					d.make_dir(entry.filename.get_base_dir())
+				f.open(entry.filename, File.WRITE)
+				f.store_buffer(buffer)
+				f.close()
+			var texture = ImageTexture.new()
+			var img : Image = Image.new()
+			img.load_png_from_buffer(buffer)
+			if entry.type == RequestType.BADGE:
+				caches[RequestType.BADGE][entry.data[0]][entry.data[1]].create_from_image(img, 0)
+			elif entry.type == RequestType.EMOTE:
+				caches[RequestType.EMOTE][entry.data[0]].create_from_image(img, 0)
+		yield(Engine.get_main_loop(), "idle_frame")
+
+# Gets badge mappings for the specified channel. Default: _global (global mappings)
+func get_badge_mapping(channel_id : String = "_global") -> Dictionary:
+	if !caches[RequestType.BADGE_MAPPING].has(channel_id):
+		var filename : String = cache_path + "/" + RequestType.keys()[RequestType.BADGE_MAPPING] + "/" + channel_id + ".json"
+		if !disk_cache && file.file_exists(filename):
+			file.open(filename, File.READ)
+			caches[RequestType.BADGE_MAPPING][channel_id] = parse_json(file.get_as_text())["badge_sets"]
+			file.close()
+		var buffer : PoolByteArray = http_request(channel_id, RequestType.BADGE_MAPPING)
+		if !buffer.empty():
+			caches[RequestType.BADGE_MAPPING][channel_id] = parse_json(buffer.get_string_from_utf8())["badge_sets"]
+			if (disk_cache):
+				file.open(filename, File.WRITE)
+				file.store_buffer(buffer)
 				file.close()
-			current = dir.get_next()
-	get_badge_mappings()
-	yield(self, "badge_mapping_available")
+		else:
+			return {}
+	return caches[RequestType.BADGE_MAPPING][channel_id]
 
-func create_request(url : String, resource : String, res_type : String) -> void:
-	var http_request = HTTPRequest.new()
-	http_request.connect("request_completed", self, "downloaded", [http_request, resource, res_type], CONNECT_ONESHOT)
-	add_child(http_request)
-	http_request.download_file = disk_cache_path + "/" + res_type + "/" + resource + ".png"
-	http_request.request(url, [], false, HTTPClient.METHOD_GET)
+func get_badge(badge_name : String, channel_id : String = "_global", scale : String = "1") -> ImageTexture:
+	var badge_data : PoolStringArray = badge_name.split("/", true, 1)
+	var texture : ImageTexture = ImageTexture.new()
+	var cachename = badge_data[0] + "_" + badge_data[1] + "_" + scale
+	var filename : String = cache_path + "/" + RequestType.keys()[RequestType.BADGE] + "/" + channel_id + "/" + cachename + ".png"
+	if !caches[RequestType.BADGE].has(channel_id):
+		caches[RequestType.BADGE][channel_id] = {}
+	if !caches[RequestType.BADGE][channel_id].has(cachename):
+		if !disk_cache && file.file_exists(filename):
+			file.open(filename, File.READ)
+			var img : Image = Image.new()
+			img.load_png_from_buffer(file.get_buffer(file.get_len()))
+			texture.create_from_image(img)
+			file.close()
+		else:
+			var map : Dictionary = caches[RequestType.BADGE_MAPPING].get(channel_id, get_badge_mapping(channel_id))
+			if !map.empty():
+				if map.has(badge_data[0]):
+					mutex.lock()
+					queue.append(Entry.new(map[badge_data[0]]["versions"][badge_data[1]]["image_url_" + scale + "x"].substr("https://static-cdn.jtvnw.net/badges/v1/".length()), RequestType.BADGE, filename, [channel_id, cachename]))
+					mutex.unlock()
+					var img = preload("res://addons/gift/placeholder.png")
+					texture.create_from_image(img)
+				elif channel_id != "_global":
+					return get_badge(badge_name, "_global", scale)
+			elif channel_id != "_global":
+				return get_badge(badge_name, "_global", scale)
+		texture.take_over_path(filename)
+		caches[RequestType.BADGE][channel_id][cachename] = texture
+	elif channel_id != "_global":
+		return get_badge(badge_name, "_global", scale)
+	return caches[RequestType.BADGE][channel_id][cachename]
 
-# Gets badge mappings for the specified channel. Empty String will get the mappings for global badges instead.
-func get_badge_mappings(channel_id : String = "") -> void:
-	var url : String
-	if(channel_id == ""):
-		channel_id = "_global"
-		url = "https://badges.twitch.tv/v1/badges/global/display"
+func get_emote(emote_id : String, scale = "1.0") -> ImageTexture:
+	var texture : ImageTexture = ImageTexture.new()
+	var cachename : String = emote_id + "_" + scale
+	var filename : String = cache_path + "/" + RequestType.keys()[RequestType.EMOTE] + "/" + cachename + ".png"
+	if !caches[RequestType.EMOTE].has(cachename):
+		if !disk_cache && file.file_exists(filename):
+			file.open(filename, File.READ)
+			var img : Image = Image.new()
+			img.load_png_from_buffer(file.get_buffer(file.get_len()))
+			texture.create_from_image(img)
+			file.close()
+		else:
+			mutex.lock()
+			queue.append(Entry.new(emote_id + "/" + scale, RequestType.EMOTE, filename, [cachename]))
+			mutex.unlock()
+			var img = preload("res://addons/gift/placeholder.png")
+			texture.create_from_image(img)
+		texture.take_over_path(filename)
+		caches[RequestType.EMOTE][cachename] = texture
+	return caches[RequestType.EMOTE][cachename]
+
+func http_request(path : String, type : int) -> PoolByteArray:
+	var error := 0
+	var buffer = PoolByteArray()
+	var new_host : String
+	match type:
+		RequestType.BADGE_MAPPING:
+			new_host = "badges.twitch.tv"
+			path = "/v1/badges/" + ("global" if path == "_global" else "channels/" + path) + "/display"
+		RequestType.BADGE, RequestType.EMOTE:
+			new_host = "static-cdn.jtvnw.net"
+			if type == RequestType.BADGE:
+				path = "/badges/v1/" + path
+			else:
+				path = "/emoticons/v1/" + path
+	if (host != new_host):
+		error = http_client.connect_to_host(new_host, 443, true)
+		while http_client.get_status() == HTTPClient.STATUS_CONNECTING or http_client.get_status() == HTTPClient.STATUS_RESOLVING:
+			http_client.poll()
+			delay(100)
+		if (error != OK):
+			print("Could not connect to " + new_host + ". Images disabled.")
+			active = false
+			return buffer
+		host = new_host
+	http_client.request(HTTPClient.METHOD_GET, path, HEADERS)
+	while (http_client.get_status() == HTTPClient.STATUS_REQUESTING):
+		http_client.poll()
+		delay(50)
+	if !(http_client.get_status() == HTTPClient.STATUS_BODY or http_client.get_status() == HTTPClient.STATUS_CONNECTED):
+		print("Request failed. Skipped " + path + " (" + RequestType.keys()[type] + ")")
+		return buffer
+	while (http_client.get_status() == HTTPClient.STATUS_BODY):
+		http_client.poll()
+		delay(1)
+		var chunk = http_client.read_response_body_chunk()
+		if (chunk.size() == 0):
+			delay(1)
+		else:
+			buffer += chunk
+	return buffer
+
+func delay(delay : int):
+	if (OS.has_feature("web")):
+		yield(Engine.get_main_loop(), "idle_frame")
 	else:
-		url = "https://badges.twitch.tv/v1/badges/channels/" + channel_id + "/display"
-	if(!badge_map.has(channel_id)):
-		var http_request = HTTPRequest.new()
-		add_child(http_request)
-		http_request.request(url, [], false, HTTPClient.METHOD_GET)
-		http_request.connect("request_completed", self, "badge_mapping_received", [http_request, channel_id], CONNECT_ONESHOT)
-	else:
-		emit_signal("badge_mapping_available")
+		OS.delay_msec(delay)
 
-func get_emote(id : String) -> ImageTexture:
-	cache_mutex.lock()
-	if(cached_images["emotes"].has(id)):
-		return cached_images["emotes"][id]
-	else:
-		create_request("http://static-cdn.jtvnw.net/emoticons/v1/" + id + "/1.0", id, "emotes")
-	cache_mutex.unlock()
-	return null
-
-func get_badge(badge_name : String, channel_id : String = "") -> ImageTexture:
-	cache_mutex.lock()
-	var badge_data : PoolStringArray = badge_name.split("/")
-	if(cached_images["badges"].has(badge_data[0])):
-		return cached_images["badges"][badge_data[0]]
-	var channel : String
-	if(!badge_map[channel_id].has(badge_data[0])):
-		channel_id = "_global"
-	create_request(badge_map[channel_id][badge_data[0]]["versions"][badge_data[1]]["image_url_1x"], badge_data[0], "badges")
-	cache_mutex.unlock()
-	return null
-
-func downloaded(result : int, response_code : int, headers : PoolStringArray, body : PoolByteArray, request : HTTPRequest, id : String, type : String) -> void:
-	if(type == "emotes"):
-		get_parent().emit_signal("emote_downloaded", id)
-	elif(type == "badges"):
-		get_parent().emit_signal("badge_downloaded", id)
-	request.queue_free()
-
-func badge_mapping_received(result : int, response_copde : int, headers : PoolStringArray, body : PoolByteArray, request : HTTPRequest, id : String) -> void:
-	badge_map[id] = parse_json(body.get_string_from_utf8())["badge_sets"]
-	if(disk_cache):
-		file.open(disk_cache_path + "/" + id + ".json", File.WRITE)
-		file.store_buffer(body)
-		file.close()
-	emit_signal("badge_mapping_available")
-	request.queue_free()
+class Entry extends Reference:
+	var path : String
+	var type : int
+	var filename : String
+	var data : Array
+	
+	func _init(path : String, type : int, filename : String, data : Array):
+		self.path = path
+		self.type = type
+		self.filename = filename
+		self.data = data

addons/gift/util/sender_data.gd

@@ -8,4 +8,4 @@ var tags : Dictionary
 func _init(usr : String, ch : String, tag_dict : Dictionary):
 	user = usr
 	channel = ch
-	tags = tag_dict
+	tags = tag_dict