«

I have created a class whose purpose is to download a list of java.net.URLs,
and put the content (of each URL) in to an array of Strings, by creating n
threads and having each thread pick up the next URL from the list, and
download the HTTP content (in a thread-safe way).
However, when it gets to the end of the list, a few threads have hung and
won't timeout. I've tried interrupting them, but that doesn't help (since
they're blocked downloading the URL, I think).

I'd like to know how to kill these threads and try again, or else is there a
safer way of downloading HTTP content from a URL?
Any ideas?

Thanks,
Alun Harford

(Note: I'm using Java 1.4.2 and Sun's JVM)

<LONG NASTY HACKY CODE>
public class ThreadedURLDownloader {
private static final int MAX_THREADS = 100;
private java.net.URL[] urls;
private int currentURL=0;
private String[] contents;
private String[] contentTypes;
private Integer[] responseCodes;
private URLDownloaderThread[] threads=new
URLDownloaderThread[MAX_THREADS];;

public ThreadedURLDownloader(java.net.URL[] urls) {
this.urls=urls;
contents=new String[urls.length];
contentTypes=new String[urls.length];
responseCodes=new Integer[urls.length];
}

public void downloadURLs() throws InterruptedException {
for(int i=0;i<MAX_THREADS;i++) {
threads[i]=new URLDownloaderThread();
threads[i].setThreadedURLDownloader(this);
threads[i].start();
}
for(int i=0;i<MAX_THREADS;i++) {
threads[i].join();
}
}

public synchronized int getNextURLNumber() throws FinishedException {
if(currentURL<=urls.length) {
System.out.println(currentURL+1);
return currentURL++;
} else {
throw new FinishedException();
}

}
public java.net.URL getURL(int index) {
return urls[index];
}
public void setContent(String content, int index) {
contents[index]=content;
}
public void setContentType(String contentType, int index) {
contentTypes[index]=contentType;
}
public void setResponseCode(int code, int index) {
responseCodes[index]=new Integer(code);
}
public String[] getContents() {
return contents;
}
}
public class URLDownloaderThread extends Thread {
ThreadedURLDownloader tud;
public URLDownloaderThread() {
super();
}
public void setThreadedURLDownloader(ThreadedURLDownloader tud) {
this.tud=tud;
}
public void run() {
try {
while(true) {
String content;
int urlIDNumber = tud.getNextURLNumber();
java.net.URL urlToDownload=new
java.net.URL("http://www.google.com");
try {
urlToDownload = tud.getURL(urlIDNumber);
} catch (ArrayIndexOutOfBoundsException aob) {
break;
}
java.net.HttpURLConnection connection =
(java.net.HttpURLConnection) urlToDownload.openConnection();
String contentType = connection.getContentType();
int responseCode;
try {
responseCode = connection.getResponseCode();
} catch (java.io.IOException e) {
content="Connection timed out";
responseCode=408;
}
if(responseCode==200) {
java.io.InputStream in = ((java.io.InputStream)
connection.getContent());
java.io.BufferedReader dis = new
java.io.BufferedReader(new java.io.InputStreamReader(in));
StringBuffer sb = new StringBuffer();
String line;
while ((line=dis.readLine()) != null) {
sb.append(line+"\n");
}
content = sb.toString();
} else {
content="No content: response was "+ responseCode;
}
tud.setContent(content,urlIDNumber);
tud.setContentType(contentType,urlIDNumber);
tud.setResponseCode(responseCode, urlIDNumber);
connection.disconnect();
connection=null;
}
} catch (java.io.IOException e) {
e.printStackTrace();
} catch (FinishedException e) {


}

}
}
</LONG NASTY HACKY CODE>