Skip to content

Commit 6dbb7e0

Browse files
mvdocmvdoc
authored
FIX avoid using jQuery for getting numpy arrays (#544)
* FIX avoid using jQuery for getting numpy arrays * Remove old offset declaration
1 parent 96bb471 commit 6dbb7e0

2 files changed

Lines changed: 94 additions & 40 deletions

File tree

‎cortex/webgl/data.py‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ class Package(object):
2121
"""Package the data into a form usable by javascript"""
2222
def __init__(self, data):
2323
self.dataset = dataset.normalize(data)
24-
self.uniques = data.uniques(collapse=True)
24+
self.uniques = list(data.uniques(collapse=True))
25+
self.subjects = set()
2526

2627
self.brains = dict()
2728
self.images = dict()
2829
for brain in self.uniques:
2930
name = brain.name
31+
self.subjects.add(brain.subject)
3032
self.brains[name] = brain.to_json(simple=True)
3133
if isinstance(brain, (dataset.Vertex, dataset.VertexRGB)):
3234
encdata = brain.vertices
@@ -60,10 +62,6 @@ def views(self):
6062
metadata.append(meta)
6163
return metadata
6264

63-
@property
64-
def subjects(self):
65-
return set(braindata.subject for braindata in self.uniques)
66-
6765
def reorder(self, subjects):
6866
indices = dict((k, np.load(os.path.splitext(v)[0]+".npz")) for k, v in subjects.items())
6967
for brain in self.uniques:

‎cortex/webgl/resources/js/datamodel.js‎

Lines changed: 91 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -84,50 +84,106 @@ NParray.fromJSON = function(json) {
8484
return NParray.fromData(data, json.dtype, json.shape);
8585
}
8686
NParray.fromURL = function(url, callback) {
87+
console.log("Loading numpy array from " + url);
8788
loadCount++;
8889
$("#dataload").show();
8990
var hideload = function() {
9091
loadCount--;
9192
if (loadCount == 0)
9293
$("#dataload").hide();
9394
};
94-
$.ajax(url, {
95-
headers: {Range: 'bytes=0-1023'},
96-
dataType: 'text',
97-
success: function(data, status, headerxhr) {
98-
if (data.slice(1, 6) != 'NUMPY')
99-
throw "Invalid npy file"
100-
console.log("npy version "+data.charCodeAt(6)+"."+data.charCodeAt(7));
101-
var nbytes = data.charCodeAt(8) + (data.charCodeAt(9) << 8);
102-
var info = parse_dict(data.slice(10, 10+nbytes));
103-
var shape = info.shape.slice(1, info.shape.length-1).split(',');
104-
shape = shape.map(function(num) { return parseInt(num.trim()) });
105-
var array = new NParray(dtypeNames[info.descr], shape);
106-
107-
if (headerxhr.status == 206) {
108-
var length = array.dtype.BYTES_PER_ELEMENT * array.size;
109-
var increment = array._slice * array.dtype.BYTES_PER_ELEMENT || length;
110-
var offset = nbytes + 10;
111-
Stream(url, length, increment, offset).progress(array.update.bind(array)).done(hideload);
112-
} else if (headerxhr.status == 200) {
113-
//Server doesn't support partial data, returned the whole thing
114-
var chars = new Uint8Array(data.length - nbytes - 10);
115-
for (var i = 0, il = chars.length; i < il; i++)
116-
chars[i] = data.charCodeAt(i+nbytes+10);
117-
array.update(chars.buffer);
118-
hideload();
119-
} else throw "Invalid response from server";
120-
callback(array);
121-
},
122-
});
95+
96+
// With thanks to https://gist.github.com/nvictus/88b3b5bfe587d32ac1ab519fd0009607
97+
var xhr = new XMLHttpRequest();
98+
xhr.open("GET", url, true);
99+
xhr.setRequestHeader("Range", "bytes=0-1023");
100+
xhr.responseType = "arraybuffer";
101+
102+
xhr.onload = function() {
103+
if (xhr.status !== 200 && xhr.status !== 206) {
104+
console.error("Failed to fetch file: status " + xhr.status);
105+
hideload();
106+
return;
107+
}
108+
109+
function asciiDecode(buf) {
110+
return String.fromCharCode.apply(null, new Uint8Array(buf));
111+
}
112+
113+
function readUint16LE(buffer) {
114+
var view = new DataView(buffer);
115+
var val = view.getUint8(0);
116+
val |= view.getUint8(1) << 8;
117+
return val;
118+
}
119+
120+
var buffer = xhr.response;
121+
console.log("Got buffer: " + buffer);
122+
console.log("Buffer length: " + buffer.byteLength);
123+
124+
// Check we got a valid npy file
125+
var magic = asciiDecode(buffer.slice(0, 6));
126+
if (magic.slice(1, 6) != 'NUMPY')
127+
throw "Invalid npy file"
128+
console.log("Magic: "+magic);
129+
130+
// OK, now let's parse the header and get some info
131+
var version = new Uint8Array(buffer.slice(6, 8));
132+
console.log("npy version " + version);
133+
134+
var headerLength = readUint16LE(buffer.slice(8, 10));
135+
console.log("Header length: " + headerLength);
136+
137+
var headerStr = asciiDecode(buffer.slice(10, 10 + headerLength));
138+
console.log("Header: " + headerStr);
139+
140+
// Create the array object
141+
var info = parse_dict(asciiDecode(buffer.slice(10, 10 + headerLength)));
142+
var shape = info.shape.slice(1, info.shape.length-1).split(',');
143+
shape = shape.map(function(num) { return parseInt(num.trim()) });
144+
var array = new NParray(dtypeNames[info.descr], shape);
145+
146+
console.log("Array shape: " + array.shape);
147+
console.log("Array size: " + array.size);
148+
console.log("Array dtype: " + array.dtype);
149+
150+
if (xhr.status == 206) {
151+
// We got partial data, we're streaming it
152+
var length = array.dtype.BYTES_PER_ELEMENT * array.size;
153+
var increment = array._slice * array.dtype.BYTES_PER_ELEMENT || length;
154+
var offset = 10 + headerLength;
155+
Stream(url, length, increment, offset).progress(array.update.bind(array)).done(hideload);
156+
} else if (xhr.status == 200) {
157+
// Server doesn't support partial data, returned the whole thing
158+
array.update(buffer.slice(10 + headerLength));
159+
hideload();
160+
}
161+
console.log("Data length: " + array.data.length);
162+
console.log("Data: " + array.data);
163+
164+
// Finish up
165+
hideload();
166+
callback(array);
167+
}
168+
169+
xhr.onerror = function() {
170+
console.error("Failed to fetch file");
171+
hideload();
172+
};
173+
174+
xhr.send();
123175
}
124176
NParray.prototype.update = function(buffer) {
125-
if (this.shape.length > 1) {
126-
this.data = new this.dtype(buffer, 0, (++this.available)*this._slice);
127-
this.loaded.notify(this.available);
128-
} else {
129-
this.data = new this.dtype(buffer);
130-
this.loaded.resolve();
177+
try {
178+
if (this.shape.length > 1) {
179+
this.data = new this.dtype(buffer, 0, (++this.available)*this._slice);
180+
this.loaded.notify(this.available);
181+
} else {
182+
this.data = new this.dtype(buffer);
183+
this.loaded.resolve();
184+
}
185+
} catch (e) {
186+
console.error("Error updating array:", e);
131187
}
132188
}
133189
NParray.prototype.view = function() {

0 commit comments

Comments
 (0)