robyww · Oct 29, 2026 · Oct 28, 2026
diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HiPSRetrieve.java b/src/firefly/java/edu/caltech/ipac/firefly/server/servlets/HiPSRetrieve.java
@@ -101,20 +101,14 @@ public static FileInfo retrieveHiPSData(URL url, String fileName, boolean always
         if (url==null) return new FileInfo(404, "Invalid URL");
 
         File targetFile = getHipsCacheFile(url, fileName);
-        var locationPrepared= prepareTargetLocation(targetFile, url, fileName);
-
-        if (!locationPrepared) {
-            return new FileInfo(HttpURLConnection.HTTP_FORBIDDEN,
-                    "this hips request conflicts with the HiPS protocol, cannot cache hips file correctly");
-        }
         boolean fileExistLocal= isHiPSFileCached(url.toString(),fileName);
 
         if (alwaysUseCached && fileExistLocal) return new FileInfo(targetFile);
 
-        // if we already have a version of the file set the download modified only option. Also set a very small timeout,
-        // so that if the server is down we don't wait long.
         try {
-            URLDownload.Options options= fileExistLocal ? URLDownload.Options.modifiedAndTimeoutOp(true,3) : URLDownload.Options.def();
+            URLDownload.Options options= URLDownload.Options.def();
+            options.setTimeoutInSec(3);
+            options.setOnlyIfModified(fileExistLocal);
             options.setDl(dl);
             FileInfo fetchedFileInfo= lockUrlDownload(url,fileName,targetFile,options);
             var rCode= fetchedFileInfo.getResponseCode();
@@ -156,7 +150,7 @@ private static boolean prepareTargetLocation(File targetFile, URL url, String fi
             logCacheConflict("parent path is a regular file: " + dir.getAbsolutePath(), url, fileName);
             if (dir.delete()) Logger.getLogger().warn("deleting file and making a directory");
         }
-        if (!dir.exists() && !dir.mkdirs()) {
+        if (!dir.mkdirs() && !dir.isDirectory()) { // another thread may create it first, e.g. sibling tiles
             return false;
         }
 
@@ -185,6 +179,11 @@ private static FileInfo lockUrlDownload(URL url, String fileName, File targetFil
         var params= new UriRefParams(url.toString());
         return LockingRetrieve.lockingRetrieve(params,
                 () -> {
+                    // inside the lock: a concurrent request for this url may be writing targetFile
+                    if (!prepareTargetLocation(targetFile, url, fileName)) {
+                        return new FileInfo(HttpURLConnection.HTTP_FORBIDDEN,
+                                "this hips request conflicts with the HiPS protocol, cannot cache hips file correctly");
+                    }
                     File f= getHipsCacheFile(url,fileName);
                     if (f!=null && f.canRead() && f.length()>minSizeCacheFile) return new FileInfo(targetFile);;
                     return URLDownload.getDataToFile(url,targetFile,null, null, options);

diff --git a/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java b/src/firefly/java/edu/caltech/ipac/firefly/server/visualize/hips/HiPSMasterList.java
@@ -170,7 +170,8 @@ public DataGroup fetchDataGroup(TableServerRequest request) throws DataAccessExc
 
             }
 
-            if (allSourceData.isEmpty()) {
+            int blankEntryCnt= hipsTable ? 1 : 0;
+            if (allSourceData.size() <= blankEntryCnt) {    // only the blank entry: every source failed, don't cache it
                 throw new IOException(errMsg);
             }
 
@@ -275,7 +276,7 @@ private void setupMeta(DataGroup dg, boolean bMulti, boolean hipsTable) {
             if (colDT.getDataType() != String.class && colDT.getDataType()!= Boolean.class) continue;
 
             if ((!bMulti && colName.equals(PARAMS.SOURCE.getKey())) || colName.equals(PARAMS.URL.getKey())) {
-                colDT.setVisibility(DataType.Visibility.hidden);
+                colDT.setVisibility(DataType.Visibility.hide);
             }
             if (colName.equals(PARAMS.HAS_MOC.getKey())) colDT.setVisibility(DataType.Visibility.hide);
             if (!hipsTable && colName.equals(PARAMS.IVOID.getKey())) colDT.setVisibility(DataType.Visibility.hide);

diff --git a/src/firefly/java/edu/caltech/ipac/util/download/ConcurrentDownload.java b/src/firefly/java/edu/caltech/ipac/util/download/ConcurrentDownload.java
@@ -54,7 +54,7 @@ public static FileInfo getData(URL url, File outfile, Map<String, String> cookie
         var status= result.getResponseCode();
 
         if (status==HttpURLConnection.HTTP_FORBIDDEN && isProbablyCloudSignedURL(url)) { // try an alternate way to get the header, may have to add for GCP as well
-            result= URLDownload.getHeader(url,cookies,newHeaders,8,true);
+            result= URLDownload.getHeader(url,cookies,newHeaders,3,true);
             status= result.getResponseCode();
         }
         if (status>=400 && (status!=HttpURLConnection.HTTP_BAD_METHOD && status!=HttpURLConnection.HTTP_FORBIDDEN)) {  // bad method or forbidden just keep going

diff --git a/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java b/src/firefly/java/edu/caltech/ipac/util/download/URLDownload.java
@@ -340,12 +340,16 @@ public static URLConnection makeConnection(URL url,
     }
 
     public static HttpURLConnection makeURLConnection(URL url,
+                                                      int timeoutInSec,
                                                       Map<String, String> cookies,
                                                       Map<String, String> requestHeaders) throws IOException {
 
         var conn= makeConnection(url, cookies, requestHeaders);
         if (conn==null) throw new IOException("HTTP connection not be created");
-        if (conn instanceof HttpURLConnection httpConn) return httpConn;
+        if (conn instanceof HttpURLConnection httpConn) {
+            if (timeoutInSec>0) httpConn.setConnectTimeout(timeoutInSec * 1000);
+            return httpConn;
+        }
         throw new IOException("HTTP connection not be created, could not be cast to HttpURLConnection");
     }
 
@@ -389,7 +393,8 @@ public static HttpResultInfo getDataFromURL(URL url,
             StopWatch.Tracker tracker = new StopWatch.Tracker("Download", null);
             tracker.starts();
             var h= buildReqHeaders(url, requestHeaders, ops);
-            conn= makeURLConnection(url,cookies,h);
+            conn= makeURLConnection(url,ops.timeoutInSec, cookies,h);
+            if (ops.readTimeoutInSec>0) conn.setReadTimeout(ops.readTimeoutInSec * 1000);
             Map<String,List<String>> reqProp= conn.getRequestProperties();
             pushPostData(conn, postData);
 
@@ -441,8 +446,9 @@ public static HttpResultInfo getHeader(URL url,
         Map <String,String> h= null;
         try {
             h= buildReqHeaders(url,requestHeaders,null);
-            HttpURLConnection conn= makeURLConnection(url,cookies,h);
-            return getHeaderFromConnection(conn,timeoutInSec,MAX_REDIRECT,cookies,h,useSmallRangeStyle);
+            HttpURLConnection conn= makeURLConnection(url,timeoutInSec, cookies,h);
+            int readTimeout= timeoutInSec>0?timeoutInSec+3:0;
+            return getHeaderFromConnection(conn,readTimeout,MAX_REDIRECT,cookies,h,useSmallRangeStyle);
         } catch (SSLException | SocketTimeoutException | UnknownHostException e) {
             return exceptionToResponse(e,h);
         } catch (IOException e) {
@@ -452,16 +458,13 @@ public static HttpResultInfo getHeader(URL url,
     }
 
     private static HttpResultInfo getHeaderFromConnection(HttpURLConnection conn,
-                                                          int timeoutInSec,
+                                                          int readTimeoutInSec,
                                                           int redirectCnt,
                                                           Map<String, String> cookies,
                                                           Map<String, String> requestHeaders,
                                                           boolean useSmallRangeStyle) throws FailedRequestException {
         try {
-            if (timeoutInSec>0) {
-                conn.setConnectTimeout(timeoutInSec * 1000);
-                conn.setReadTimeout(timeoutInSec * 1000 + 3000);
-            }
+            if (readTimeoutInSec>0) conn.setReadTimeout(readTimeoutInSec * 1000);
 
             if (useSmallRangeStyle) {
                 requestHeaders= new HashMap<>(requestHeaders);
@@ -495,8 +498,8 @@ private static HttpResultInfo getHeaderFromConnection(HttpURLConnection conn,
             var responseCode= getResponseCode(conn);
             if (responseCode >= 300 && responseCode < 400) {
                 if (redirectCnt > 0 && Arrays.asList(301,302,303,307,308).contains(responseCode)) {
-                    HttpURLConnection newConn = makeURLConnection(urlFromLocation(conn), cookies, requestHeaders);
-                    return getHeaderFromConnection(newConn, 2, redirectCnt-1, cookies, requestHeaders,useSmallRangeStyle);
+                    HttpURLConnection newConn = makeURLConnection(urlFromLocation(conn), 2, cookies, requestHeaders);
+                    return getHeaderFromConnection(newConn, 5, redirectCnt-1, cookies, requestHeaders,useSmallRangeStyle);
                 }
                 result.putAttribute("Location", conn.getHeaderField("Location"));
                 throw new FailedRequestException(ResponseMessage.getHttpResponseMessage(responseCode),
@@ -531,7 +534,7 @@ public static FileInfo getDataToFileUsingPost(URL url, Map<String,?> postData,
                                                   int timeoutInSec) throws FailedRequestException {
         try {
             Options ops= new Options(true, true, 0L, false, false, timeoutInSec, dl, false, false);
-            return getDataToFile(makeURLConnection(url, cookies, requestHeader), outfile, ops, postData,0);
+            return getDataToFile(makeURLConnection(url, timeoutInSec, cookies, requestHeader), outfile, ops, postData,0);
         } catch (IOException e) {
             loggerError(url, postData, e);
             throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e);
@@ -581,7 +584,7 @@ public static FileInfo getDataToFile(URL url,
                                          Options ops) throws FailedRequestException {
         try {
             var h= buildReqHeaders(url,requestHeaders,ops);
-            return getDataToFile(makeURLConnection(url, cookies, h), outfile, ops, null, ops.allowRedirect?MAX_REDIRECT:0);
+            return getDataToFile(makeURLConnection(url, ops.timeoutInSec, cookies, h), outfile, ops, null, ops.allowRedirect?MAX_REDIRECT:0);
         } catch (IOException e) {
             throw new FailedRequestException(ResponseMessage.getNetworkCallFailureMessage(e), e);
         }
@@ -613,10 +616,7 @@ public static FileInfo getDataToFile(HttpURLConnection conn,
             Map<String, List<String>> sendHeaders = null;
             long start = System.currentTimeMillis();
             try {
-                if (ops.timeoutInSec > 0) {
-                    conn.setConnectTimeout(ops.timeoutInSec * 1000);//Sets a specified timeout value, in milliseconds
-                    conn.setReadTimeout(ops.timeoutInSec * 1000);
-                }
+                if (ops.readTimeoutInSec>0) conn.setReadTimeout(ops.readTimeoutInSec * 1000);
                 pushPostData(conn, postData);
                 sendHeaders = conn.getRequestProperties();
                 if (ops.onlyIfModified) {
@@ -678,7 +678,7 @@ private static FileInfo redirect(HttpURLConnection conn,
 
         var ignore= outfile.delete();
         String urlStr= conn.getHeaderField("Location");
-        HttpURLConnection newConn= makeURLConnection(makeURL(urlStr), null, null);
+        HttpURLConnection newConn= makeURLConnection(makeURL(urlStr), ops.timeoutInSec, null, null);
         for(Map.Entry<String,List<String>> entry : reqProp.entrySet()) {
             for(String s : entry.getValue()) newConn.setRequestProperty(entry.getKey(), s);
         }
@@ -1054,6 +1054,7 @@ public static class Options {
         private boolean allowRedirect;
         private boolean useCredentials;
         private int timeoutInSec;
+        private int readTimeoutInSec;     // 0: no read timeout, wait as long as the server keeps the connection open
         private DownloadListener dl;
         private boolean logErrorsOnly;
         private boolean expectStaticFile;
@@ -1082,24 +1083,13 @@ public Options(boolean onlyIfModified, boolean uncompress, long maxFileSize, boo
 
 
 
-        /**
-         * convenience function
-         * set no size limit,
-         * sets true: uncompress, allowRedirect, use credentials
-         * @param onlyIfModified - check for file modification
-         * @param timeoutInSec - timeout in seconds, 0 use the default timeout
-         * @return Options
-         */
-        public static Options modifiedAndTimeoutOp(boolean onlyIfModified, int timeoutInSec) {
-            return new Options(onlyIfModified, true, 0, true, true, timeoutInSec, null, false, false);
-        }
-
         public void setOnlyIfModified(boolean onlyIfModified) { this.onlyIfModified = onlyIfModified; }
         public void setUncompress(boolean uncompress) { this.uncompress = uncompress; }
         public void setMaxFileSize(long maxFileSize) { this.maxFileSize = maxFileSize; }
         public void setAllowRedirect(boolean allowRedirect) { this.allowRedirect = allowRedirect; }
         public void setUseCredentials(boolean useCredentials) { this.useCredentials = useCredentials; }
         public void setTimeoutInSec(int timeoutInSec) { this.timeoutInSec = timeoutInSec; }
+        public void setReadTimeoutInSec(int readTimeoutInSec) { this.readTimeoutInSec = readTimeoutInSec; }
         public void setDl(DownloadListener dl) { this.dl = dl; }
         public void setLogErrorsOnly(boolean logErrorsOnly) { this.logErrorsOnly = logErrorsOnly; }
         public void setExpectStaticFile(boolean expectStaticFile) { this.expectStaticFile = expectStaticFile; }

diff --git a/src/firefly/js/charts/Colorscale.js b/src/firefly/js/charts/Colorscale.js
@@ -1,5 +1,3 @@
-//import Color from '../util/Color.js';
-
 // Sequential color scales from colorlover
 // roughly matching the first 8 trace colors:
 // '#1f77b4', '#2ca02c', '#d62728', '#9467bd',
@@ -104,17 +102,4 @@ export function colorscaleNameToVal(name) {
     }
     return undefined;
 
-}
-
-// does not work well
-// function createColorScale(baseColor) {
-//     const numcols = 5;
-//     return [[0.0, 'rgb(237, 237, 237)']].concat(
-//         Color.makeSimpleColorMap(baseColor, numcols, true).map((e,i) => {
-//             const [r, g, b] = Color.toRGB(e);
-//             let p = 1.0;
-//             if (i < (numcols-1)) p = (i+1)*(1/numcols);
-//             return [p, `rgb(${r},${g},${b})`];
-//         })
-//     );
-// }
+}
diff --git a/src/firefly/js/util/Color.js b/src/firefly/js/util/Color.js
@@ -50,19 +50,6 @@ export const brighter = (colorStr,level=1) =>
 export const darker = (colorStr,level=1) =>
     chroma.valid(colorStr) ? chroma(colorStr).darken(level).toString() : undefined;
 
-/**
- * Make a simple color map array of no more than 10 entries.  Map goes from darker to brighter.
- * @param baseColor
- * @param mapSize
- * @param reverse
- * @return {*}
- */
-function makeSimpleColorMap(baseColor, mapSize=10, reverse=false) {
-    const c1= chroma(baseColor).brighten(2);
-    const c2= chroma(baseColor).darken(1);
-    const cAry= reverse ? [c2,c1] : [c1,c2];
-   return chroma.scale(cAry).colors(mapSize);
-}
 
 const [R, G, B, A] = [0, 1, 2, 3];
 const [LUMI, CONTRAST, HSP] = [0, 1, 2];
@@ -71,9 +58,6 @@ const [LUMI, CONTRAST, HSP] = [0, 1, 2];
 
 export function rateOpacity(color, ratio) {
     const rgba = getRGBA(color);
-    if (!rgba) {
-        return 'rgba(255, 255, 255,' + (1.0*ratio) + ')';
-    }
     const newAlpha = Math.max(Math.min(rgba[3] * ratio, 1.0), 0.0);
     const newColor = rgba.slice(R, A);
     newColor.push(newAlpha);
@@ -95,10 +79,6 @@ export function getBWBackground(color, method = LUMI) {
     let luma;
 
     const rgba = getRGBA(color);
-    if (!rgba) {
-        return 'rgba(255, 255, 255, 1.0)';
-    }
-
     const linearLuma = (w) => {
         return [R, G, B].reduce((lum, c) => {
             lum += w[c] * rgba[c];
@@ -250,18 +230,18 @@ function getComplementaryColor(color) {
 }
 
 export function toRGBAString(rgba) {
-    const len = (!rgba || !isArray(rgba)) ? 0 : rgba.length;
-    const val = len === 0 ? 255 : rgba[len-1];
-
+    if (!isArray(rgba) || !rgba.length) return '';
+    const rgbaOut= [...rgba];
+    const len= rgbaOut.length;
     if (len <= A) {
-        if (len === 0) rgba = [];
+        const lastValue= rgbaOut[len-1];
         for (let i = 0; i < A - len; i++ ) {
-            rgba.push(val);
+            rgbaOut.push(lastValue);
         }
-        rgba.push(1.0);
+        rgbaOut.push(1.0);
     }
 
-    return `rgba(${rgba[R]}, ${rgba[G]}, ${rgba[B]}, ${rgba[A]})`;
+    return `rgba(${rgbaOut[R]}, ${rgbaOut[G]}, ${rgbaOut[B]}, ${rgbaOut[A]})`;
 }
 
 /*